Verified Commit fd2255c2 authored by Geoff Pado's avatar Geoff Pado

Translate text observations to rectangles in image coordinates

parent 053682f9
......@@ -30,6 +30,10 @@
041EFF72225C3DC00058D8EE /* PhotoLibraryDataSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = 041EFF71225C3DC00058D8EE /* PhotoLibraryDataSource.swift */; };
041EFF74225C3DD20058D8EE /* PhotoLibraryViewCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = 041EFF73225C3DD20058D8EE /* PhotoLibraryViewCell.swift */; };
041EFF76225C3DF10058D8EE /* PhotoLibraryViewLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = 041EFF75225C3DF10058D8EE /* PhotoLibraryViewLayout.swift */; };
043CD9C9226EB0400012F5AE /* TextRectangleDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = 043CD9C8226EB0400012F5AE /* TextRectangleDetector.swift */; };
043CD9CB226EB0830012F5AE /* TextRectangleDetectionOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 043CD9CA226EB0830012F5AE /* TextRectangleDetectionOperation.swift */; };
043CD9CF226EB0C70012F5AE /* DetectedTextObservation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 043CD9CE226EB0C70012F5AE /* DetectedTextObservation.swift */; };
043CD9D1226EB0F60012F5AE /* UIImageExtensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 043CD9D0226EB0F60012F5AE /* UIImageExtensions.swift */; };
047072932268134500FF20B6 /* PhotoEditingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 047072922268134500FF20B6 /* PhotoEditingView.swift */; };
047072952268137900FF20B6 /* PhotoEditingImageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 047072942268137900FF20B6 /* PhotoEditingImageView.swift */; };
048909F2226571AB0048E203 /* PhotoEditingViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 048909F1226571AB0048E203 /* PhotoEditingViewController.swift */; };
......@@ -80,6 +84,10 @@
041EFF71225C3DC00058D8EE /* PhotoLibraryDataSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhotoLibraryDataSource.swift; sourceTree = "<group>"; };
041EFF73225C3DD20058D8EE /* PhotoLibraryViewCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhotoLibraryViewCell.swift; sourceTree = "<group>"; };
041EFF75225C3DF10058D8EE /* PhotoLibraryViewLayout.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhotoLibraryViewLayout.swift; sourceTree = "<group>"; };
043CD9C8226EB0400012F5AE /* TextRectangleDetector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TextRectangleDetector.swift; sourceTree = "<group>"; };
043CD9CA226EB0830012F5AE /* TextRectangleDetectionOperation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TextRectangleDetectionOperation.swift; sourceTree = "<group>"; };
043CD9CE226EB0C70012F5AE /* DetectedTextObservation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DetectedTextObservation.swift; sourceTree = "<group>"; };
043CD9D0226EB0F60012F5AE /* UIImageExtensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UIImageExtensions.swift; sourceTree = "<group>"; };
047072922268134500FF20B6 /* PhotoEditingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhotoEditingView.swift; sourceTree = "<group>"; };
047072942268137900FF20B6 /* PhotoEditingImageView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhotoEditingImageView.swift; sourceTree = "<group>"; };
048909F1226571AB0048E203 /* PhotoEditingViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhotoEditingViewController.swift; sourceTree = "<group>"; };
......@@ -197,10 +205,21 @@
path = "Photo Selection";
sourceTree = "<group>";
};
043CD9CD226EB0B20012F5AE /* Text Detection */ = {
isa = PBXGroup;
children = (
043CD9C8226EB0400012F5AE /* TextRectangleDetector.swift */,
043CD9CA226EB0830012F5AE /* TextRectangleDetectionOperation.swift */,
043CD9CE226EB0C70012F5AE /* DetectedTextObservation.swift */,
);
path = "Text Detection";
sourceTree = "<group>";
};
048909EF2265716B0048E203 /* Photo Editing */ = {
isa = PBXGroup;
children = (
048909F1226571AB0048E203 /* PhotoEditingViewController.swift */,
043CD9CD226EB0B20012F5AE /* Text Detection */,
047072922268134500FF20B6 /* PhotoEditingView.swift */,
047072942268137900FF20B6 /* PhotoEditingImageView.swift */,
048909F3226573750048E203 /* PhotoEditorPresenting.swift */,
......@@ -224,6 +243,7 @@
children = (
041EFF182251AAFE0058D8EE /* UIViewControllerExtensions.swift */,
04D68BC4225EE2C000D09BBD /* GeometryExtensions.swift */,
043CD9D0226EB0F60012F5AE /* UIImageExtensions.swift */,
);
path = Extensions;
sourceTree = "<group>";
......@@ -379,9 +399,11 @@
041EFF5D2255A85F0058D8EE /* NavigationController.swift in Sources */,
041EFF492252FAF50058D8EE /* IntroViewController.swift in Sources */,
047072952268137900FF20B6 /* PhotoEditingImageView.swift in Sources */,
043CD9CF226EB0C70012F5AE /* DetectedTextObservation.swift in Sources */,
047072932268134500FF20B6 /* PhotoEditingView.swift in Sources */,
041EFF6B225C36350058D8EE /* PhotoLibraryViewController.swift in Sources */,
041EFF70225C3D830058D8EE /* PhotoLibraryView.swift in Sources */,
043CD9C9226EB0400012F5AE /* TextRectangleDetector.swift in Sources */,
048909F2226571AB0048E203 /* PhotoEditingViewController.swift in Sources */,
041EFF552255A0B00058D8EE /* Colors.swift in Sources */,
041EFF72225C3DC00058D8EE /* PhotoLibraryDataSource.swift in Sources */,
......@@ -397,10 +419,12 @@
04D68BC7225EE2FD00D09BBD /* PhotoLibraryViewCellImageView.swift in Sources */,
041EFF5B2255A39D0058D8EE /* Fonts.swift in Sources */,
041EFF50225303120058D8EE /* PromptLabel.swift in Sources */,
043CD9D1226EB0F60012F5AE /* UIImageExtensions.swift in Sources */,
041EFF1B2251AB8D0058D8EE /* AppWindow.swift in Sources */,
041EFF4E225302CC0058D8EE /* IntroView.swift in Sources */,
041EFF74225C3DD20058D8EE /* PhotoLibraryViewCell.swift in Sources */,
04D68BC5225EE2C000D09BBD /* GeometryExtensions.swift in Sources */,
043CD9CB226EB0830012F5AE /* TextRectangleDetectionOperation.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
......
// Created by Geoff Pado on 4/22/19.
// Copyright © 2019 Cocoatype, LLC. All rights reserved.
import UIKit
extension UIImage.Orientation {
var cgImagePropertyOrientation: CGImagePropertyOrientation {
switch self {
case .up: return .up
case .down: return .down
case .left: return .left
case .right: return .right
case .upMirrored: return .upMirrored
case .downMirrored: return .downMirrored
case .leftMirrored: return .leftMirrored
case .rightMirrored: return .rightMirrored
@unknown default: return .up
}
}
}
// Created by Geoff Pado on 4/15/19.
// Copyright © 2019 Cocoatype, LLC. All rights reserved.
import os.log
import Photos
import UIKit
import Vision
class PhotoEditingViewController: UIViewController {
init(asset: PHAsset) {
......@@ -53,108 +51,3 @@ class PhotoEditingViewController: UIViewController {
fatalError("\(className) does not implement init(coder:)")
}
}
class TextRectangleDetector: NSObject {
func detectTextRectangles(in image: UIImage, completionHandler: (([VNTextObservation]?) -> Void)? = nil) {
guard let detectionOperation = TextRectangleDetectionOperation(image: image) else {
completionHandler?(nil)
return
}
detectionOperation.completionBlock = { [weak detectionOperation] in
completionHandler?(detectionOperation?.textRectangleResults)
}
operationQueue.addOperation(detectionOperation)
}
// MARK: Boilerplate
private let operationQueue = OperationQueue()
}
class TextRectangleDetectionOperation: Operation {
init?(image: UIImage) {
guard let cgImage = image.cgImage else { return nil }
self.imageRequestHandler = VNImageRequestHandler(cgImage: cgImage, orientation: image.imageOrientation.cgImagePropertyOrientation)
super.init()
}
var textRectangleResults: [VNTextObservation]?
override func start() {
let imageRequest = VNDetectTextRectanglesRequest { [weak self] request, error in
guard let textObservations = (request.results as? [VNTextObservation]) else {
TextRectangleDetectionOperation.log("error getting text rectangles: \(error?.localizedDescription ?? "(null)")", type: .error)
self?._finished = true
self?._executing = false
return
}
self?.textRectangleResults = textObservations
self?._finished = true
self?._executing = false
}
do {
try imageRequestHandler.perform([imageRequest])
_executing = true
} catch {
TextRectangleDetectionOperation.log("error starting image request: \(error.localizedDescription)", type: .error)
_finished = true
_executing = false
}
}
// MARK: Logging
static var log: OSLog { return OSLog(subsystem: "com.cocoatype.Highlighter", category: "Text Detection") }
static func log(_ text: String, type: OSLogType = .default) {
os_log("%@", log: TextRectangleDetectionOperation.log, type: type, text)
}
// MARK: Boilerplate
private let imageRequestHandler: VNImageRequestHandler
override var isAsynchronous: Bool { return true }
private var _executing = false {
willSet {
willChangeValue(for: \.isExecuting)
}
didSet {
didChangeValue(for: \.isExecuting)
}
}
override var isExecuting: Bool { return _executing }
private var _finished = false {
willSet {
willChangeValue(for: \.isFinished)
}
didSet {
didChangeValue(for: \.isFinished)
}
}
override var isFinished: Bool { return _finished }
}
extension UIImage.Orientation {
var cgImagePropertyOrientation: CGImagePropertyOrientation {
switch self {
case .up: return .up
case .down: return .down
case .left: return .left
case .right: return .right
case .upMirrored: return .upMirrored
case .downMirrored: return .downMirrored
case .leftMirrored: return .leftMirrored
case .rightMirrored: return .rightMirrored
@unknown default: return .up
}
}
}
// Created by Geoff Pado on 4/22/19.
// Copyright © 2019 Cocoatype, LLC. All rights reserved.
import UIKit
import Vision
struct DetectedTextObservation {
init(_ textObservation: VNTextObservation, in image: UIImage) {
var boundingBox = textObservation.boundingBox
boundingBox.origin.y = (1.0 - boundingBox.origin.y)
let imageSize = image.size * image.scale
boundingBox.origin.x *= imageSize.width
boundingBox.origin.y *= imageSize.height
boundingBox.size.width *= imageSize.width
boundingBox.size.height *= imageSize.height
boundingBox.origin.y -= boundingBox.size.height
self.bounds = boundingBox.integral
}
let bounds: CGRect
}
// Created by Geoff Pado on 4/22/19.
// Copyright © 2019 Cocoatype, LLC. All rights reserved.
import os.log
import UIKit
import Vision
class TextRectangleDetectionOperation: Operation {
init?(image: UIImage) {
guard let cgImage = image.cgImage else { return nil }
self.imageRequestHandler = VNImageRequestHandler(cgImage: cgImage, orientation: image.imageOrientation.cgImagePropertyOrientation)
super.init()
}
var textRectangleResults: [VNTextObservation]?
override func start() {
let imageRequest = VNDetectTextRectanglesRequest { [weak self] request, error in
guard let textObservations = (request.results as? [VNTextObservation]) else {
TextRectangleDetectionOperation.log("error getting text rectangles: \(error?.localizedDescription ?? "(null)")", type: .error)
self?._finished = true
self?._executing = false
return
}
self?.textRectangleResults = textObservations
self?._finished = true
self?._executing = false
}
do {
try imageRequestHandler.perform([imageRequest])
_executing = true
} catch {
TextRectangleDetectionOperation.log("error starting image request: \(error.localizedDescription)", type: .error)
_finished = true
_executing = false
}
}
// MARK: Logging
static var log: OSLog { return OSLog(subsystem: "com.cocoatype.Highlighter", category: "Text Detection") }
static func log(_ text: String, type: OSLogType = .default) {
os_log("%@", log: TextRectangleDetectionOperation.log, type: type, text)
}
// MARK: Boilerplate
private let imageRequestHandler: VNImageRequestHandler
override var isAsynchronous: Bool { return true }
private var _executing = false {
willSet {
willChangeValue(for: \.isExecuting)
}
didSet {
didChangeValue(for: \.isExecuting)
}
}
override var isExecuting: Bool { return _executing }
private var _finished = false {
willSet {
willChangeValue(for: \.isFinished)
}
didSet {
didChangeValue(for: \.isFinished)
}
}
override var isFinished: Bool { return _finished }
}
// Created by Geoff Pado on 4/22/19.
// Copyright © 2019 Cocoatype, LLC. All rights reserved.
import UIKit
class TextRectangleDetector: NSObject {
func detectTextRectangles(in image: UIImage, completionHandler: (([DetectedTextObservation]?) -> Void)? = nil) {
guard let detectionOperation = TextRectangleDetectionOperation(image: image) else {
completionHandler?(nil)
return
}
detectionOperation.completionBlock = { [weak detectionOperation] in
let detectedTextObservations = detectionOperation?.textRectangleResults?.map { DetectedTextObservation($0, in: image) }
completionHandler?(detectedTextObservations)
}
operationQueue.addOperation(detectionOperation)
}
// MARK: Boilerplate
private let operationQueue = OperationQueue()
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment