diff --git a/README.md b/README.md index 6e20886..3f53128 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,25 @@ Now the Bluetooth server starts when the Pi boots. Also, Bluetooth has to be in The application has three tabs. The first tab is the main view. Here the rider can see alerts when an object is approaching (TBD). The next view is the Live Streaming. The app will connect to the Pi over Wi-Fi and stream a live feed from the camera. The user has the option to record the streaming and to use ML to detect the objects in the feed. The last tab is the settings tab (TBD). At the top of the screen there are two icons that show if the Bluetooth and Wi-Fi connections are available by turning green. +## Disabling Unnecessary Bluetooth Services + +By default Raspbian's bluetooth service includes AVRCP, which is for the capability of streaming audio to the raspberry pi, but we do not require this. Also, the bluetooth service includes SAP (Sim Access Profile) which we do not make use of. Do the following to disable those features: + +```bash +sudo systemctl edit bluetooth.service +``` + +Then, add the following lines to the indicated regions. + +```conf +[Service] +# First clear the responsible variable. +ExecStart= +# Then reassign it with the aforemention plugins disabled. +ExecStart=/usr/libexec/bluetooth/bluetoothd --noplugin=sap,avrcp +``` + + ## Contributing ### Updating requirements.txt diff --git a/Rear Rider/Rear Rider.xcodeproj/project.pbxproj b/Rear Rider/Rear Rider.xcodeproj/project.pbxproj index e9dcc51..82388a4 100644 --- a/Rear Rider/Rear Rider.xcodeproj/project.pbxproj +++ b/Rear Rider/Rear Rider.xcodeproj/project.pbxproj @@ -47,6 +47,7 @@ C36AA2E728FC80B2001F24A3 /* ShareSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = C36AA2E428FC80B2001F24A3 /* ShareSheet.swift */; }; C36AA2E828FC80B2001F24A3 /* RecordingExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = C36AA2E528FC80B2001F24A3 /* RecordingExtension.swift */; }; C36AA2EC28FC8278001F24A3 /* RiderView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C36AA2EB28FC8278001F24A3 /* RiderView.swift */; }; + C39C7C2D2933C2AD007A5884 /* beep.mp3 in Resources */ = {isa = PBXBuildFile; fileRef = C39C7C2C2933C2AD007A5884 /* beep.mp3 */; }; C3A64EB329172F2A0067FDF0 /* BLETests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3A64EB229172F2A0067FDF0 /* BLETests.swift */; }; C3BD658328E8DB3B0042CEB4 /* BLEManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3BD658228E8DB3B0042CEB4 /* BLEManager.swift */; }; C3FC5C66290C64B800929D1D /* yolov5m.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = C3FC5C65290C64B800929D1D /* yolov5m.mlmodel */; }; @@ -98,6 +99,7 @@ C36AA2E428FC80B2001F24A3 /* ShareSheet.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ShareSheet.swift; sourceTree = ""; }; C36AA2E528FC80B2001F24A3 /* RecordingExtension.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RecordingExtension.swift; sourceTree = ""; }; C36AA2EB28FC8278001F24A3 /* RiderView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RiderView.swift; sourceTree = ""; }; + C39C7C2C2933C2AD007A5884 /* beep.mp3 */ = {isa = PBXFileReference; lastKnownFileType = audio.mp3; path = beep.mp3; sourceTree = ""; }; C3A64EB229172F2A0067FDF0 /* BLETests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = BLETests.swift; sourceTree = ""; }; C3BD658228E8DB3B0042CEB4 /* BLEManager.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = BLEManager.swift; sourceTree = ""; }; C3FC5C65290C64B800929D1D /* yolov5m.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; path = yolov5m.mlmodel; sourceTree = ""; }; @@ -182,6 +184,7 @@ 4845584329089489002D81FC /* Alerts */ = { isa = PBXGroup; children = ( + C39C7C2C2933C2AD007A5884 /* beep.mp3 */, 48CB852E2902EF5200283DBD /* vehicleAlert.mp3 */, 48CB852C2902E60700283DBD /* RearRiderAlerts.swift */, ); @@ -380,6 +383,7 @@ buildActionMask = 2147483647; files = ( 4851CE1028E49485003B5B76 /* Preview Assets.xcassets in Resources */, + C39C7C2D2933C2AD007A5884 /* beep.mp3 in Resources */, 483E42B4291BE71800B9A3C3 /* GoogleService-Info.plist in Resources */, 4851CE0D28E49485003B5B76 /* Assets.xcassets in Resources */, 48CB852F2902EF5200283DBD /* vehicleAlert.mp3 in Resources */, @@ -574,7 +578,7 @@ CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"Rear Rider/Preview Content\""; - DEVELOPMENT_TEAM = 5KVT845VZL; + DEVELOPMENT_TEAM = T53AYVVL5N; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSBluetoothAlwaysUsageDescription = "This app requires Bluetooth for connecting to RaspberryPi"; @@ -588,7 +592,7 @@ "@executable_path/Frameworks", ); MARKETING_VERSION = 1.0; - PRODUCT_BUNDLE_IDENTIFIER = RearRider; + PRODUCT_BUNDLE_IDENTIFIER = com.Calin.RearRider; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; @@ -604,7 +608,7 @@ CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"Rear Rider/Preview Content\""; - DEVELOPMENT_TEAM = 5KVT845VZL; + DEVELOPMENT_TEAM = T53AYVVL5N; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSBluetoothAlwaysUsageDescription = "This app requires Bluetooth for connecting to RaspberryPi"; @@ -618,7 +622,7 @@ "@executable_path/Frameworks", ); MARKETING_VERSION = 1.0; - PRODUCT_BUNDLE_IDENTIFIER = RearRider; + PRODUCT_BUNDLE_IDENTIFIER = com.Calin.RearRider; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; diff --git a/Rear Rider/Rear Rider.xcodeproj/project.xcworkspace/xcuserdata/lydiapescaru.xcuserdatad/UserInterfaceState.xcuserstate b/Rear Rider/Rear Rider.xcodeproj/project.xcworkspace/xcuserdata/lydiapescaru.xcuserdatad/UserInterfaceState.xcuserstate index 786e691..cddf4a0 100644 Binary files a/Rear Rider/Rear Rider.xcodeproj/project.xcworkspace/xcuserdata/lydiapescaru.xcuserdatad/UserInterfaceState.xcuserstate and b/Rear Rider/Rear Rider.xcodeproj/project.xcworkspace/xcuserdata/lydiapescaru.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/Rear Rider/Rear Rider/Alerts/RearRiderAlerts.swift b/Rear Rider/Rear Rider/Alerts/RearRiderAlerts.swift index 66ac490..ef4f74c 100644 --- a/Rear Rider/Rear Rider/Alerts/RearRiderAlerts.swift +++ b/Rear Rider/Rear Rider/Alerts/RearRiderAlerts.swift @@ -13,15 +13,19 @@ enum AlertErrors: Error { case fileNotFound(String) } - /** * Class for managing any audio and visual alerts for the rider */ class RearRiderAlerts: ObservableObject { - var mLModel = ImageIdentification() + var mLModel = ImageIdentification.shared var player: AVAudioPlayer! var soundFile: URL! = nil + var unsafe_distance: Int = 900 + @Published var distance: Int = 0 + var alert_enabled: Bool = true + var vehicles_only: Bool = true + @Published var frame: UIImage = UIImage() static var shared = RearRiderAlerts() @@ -29,6 +33,17 @@ class RearRiderAlerts: ObservableObject { var pic_size:Int = 0 var pic_first_time:Bool = true + init() { + do { + try AVAudioSession.sharedInstance().setCategory( + AVAudioSession.Category.multiRoute, // this setting allows the sound to be played on the speaker instead of Bluetooth + options: AVAudioSession.CategoryOptions.duckOthers) + + try AVAudioSession.sharedInstance().setActive(true) + } catch let error { + print(error) + } + } /// When this is set, the transfer of the packets will commence var pic_packets:Int = 0 { @@ -46,7 +61,7 @@ class RearRiderAlerts: ObservableObject { mLModel.detectObjects(image: frame) pic_first_time = true picData = NSMutableData() - if mLModel.detected_objs.isEmpty { askForPic() } // if no objects detect ask for another pic + if mLModel.bndRects.isEmpty { askForPic() } // if no objects detect ask for another pic } else { BLEManager.shared.getPicPacket(index: packet_recv) @@ -79,24 +94,30 @@ class RearRiderAlerts: ObservableObject { func playAudioAlert() { // do nothing if we don't have a sound file configured if soundFile == nil { return } - - do { - try AVAudioSession.sharedInstance().setCategory( - AVAudioSession.Category.playback, - options: AVAudioSession.CategoryOptions.duckOthers - ) - - try AVAudioSession.sharedInstance().setActive(true) - player.play() - } catch let error { - print(error) - } + player.play() } /// Asks the RPi for the picture's metadata (size and number of packets) func askForPic() { - mLModel.detected_objs.removeAll() + //mLModel.detected_objs.removeAll() mLModel.clearBndRects() BLEManager.shared.getPicInfo() } + + /// Play an alert sound when necessary + /// - Parameter d: Distance of the object detected by LiDAR + func alert_rider(distance d: Int) { + self.distance = d + + if d <= unsafe_distance && alert_enabled { + if vehicles_only { + if mLModel.checkForVehicles() { + playAudioAlert() + } + } + else { + playAudioAlert() + } + } + } } diff --git a/Rear Rider/Rear Rider/Alerts/beep.mp3 b/Rear Rider/Rear Rider/Alerts/beep.mp3 new file mode 100644 index 0000000..cd3a3ec Binary files /dev/null and b/Rear Rider/Rear Rider/Alerts/beep.mp3 differ diff --git a/Rear Rider/Rear Rider/Assets.xcassets/AppIcon.appiconset/Contents.json b/Rear Rider/Rear Rider/Assets.xcassets/AppIcon.appiconset/Contents.json index 9221b9b..5a1f9ff 100644 --- a/Rear Rider/Rear Rider/Assets.xcassets/AppIcon.appiconset/Contents.json +++ b/Rear Rider/Rear Rider/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -36,6 +36,7 @@ "size" : "60x60" }, { + "filename" : "rearrider_180x180.png", "idiom" : "iphone", "scale" : "3x", "size" : "60x60" diff --git a/Rear Rider/Rear Rider/Assets.xcassets/AppIcon.appiconset/rearrider_180x180.png b/Rear Rider/Rear Rider/Assets.xcassets/AppIcon.appiconset/rearrider_180x180.png new file mode 100644 index 0000000..b69f056 Binary files /dev/null and b/Rear Rider/Rear Rider/Assets.xcassets/AppIcon.appiconset/rearrider_180x180.png differ diff --git a/Rear Rider/Rear Rider/BlueTooth/BLEManager.swift b/Rear Rider/Rear Rider/BlueTooth/BLEManager.swift index d1145cf..702a575 100644 --- a/Rear Rider/Rear Rider/BlueTooth/BLEManager.swift +++ b/Rear Rider/Rear Rider/BlueTooth/BLEManager.swift @@ -24,6 +24,7 @@ struct CBUUIDs { static let BLEConfigCharacteristicUUID = CBUUID(string: "501beabd-3f66-4cca-ba7a-0fbf4f81870c") static let BLEWifiCharacteristicUUID = CBUUID(string: "cd41b278-6254-4c89-9cd1-fd2578ab8fcc") static let BLEPictureCharacteristicUUID = CBUUID(string: "cd41b278-6254-4c89-9cd1-fd2578ab8abb") + static let BLELiDARCharacteristicUUID = CBUUID(string: "92cb916f-d996-4f30-8cba-cf3ab8aede56") } /// The purpose of this class is to set the iPhone as a central manager and connect to the RaspberryPi as a peripheral @@ -36,6 +37,7 @@ class BLEManager: NSObject, ObservableObject, CBCentralManagerDelegate, CBPeriph private var configCharacteristic: CBCharacteristic! private var wifiCharacteristic: CBCharacteristic! private var picCharacteristic: CBCharacteristic! + private var lidarCharacteristic: CBCharacteristic! //mostly for testing purposes var ConfigCharacteristic: CBCharacteristic { @@ -217,6 +219,12 @@ class BLEManager: NSObject, ObservableObject, CBCentralManagerDelegate, CBPeriph log.addLog(from: "BT", message: "Picture Characteristic set") connected = true } + else if characteristic.uuid.isEqual(CBUUIDs.BLELiDARCharacteristicUUID) { + lidarCharacteristic = characteristic + log.addLog(from: "BT", message: "LiDAR Characteristic set") + peripheral.setNotifyValue(true, for: lidarCharacteristic) + connected = true + } } } @@ -267,6 +275,13 @@ class BLEManager: NSObject, ObservableObject, CBCentralManagerDelegate, CBPeriph } } } + else if characteristic == lidarCharacteristic { + let d = String(data: characteristic.value ?? Data(), encoding: String.Encoding.utf8) + if d != nil { + let distance: Int = Int(d ?? "0") ?? 0 + RearRiderAlerts.shared.alert_rider(distance: distance) + } + } } /// Called when the state of the connection changes @@ -276,6 +291,7 @@ class BLEManager: NSObject, ObservableObject, CBCentralManagerDelegate, CBPeriph func peripheral(_ peripheral: CBPeripheral, didModifyServices invalidatedServices: [CBService]) { connected = false log.addLog(from: "BT", message: "RPi disconnected") + startScanning() // try to reconnect } /// Called when the peripheral disconnects @@ -289,6 +305,7 @@ class BLEManager: NSObject, ObservableObject, CBCentralManagerDelegate, CBPeriph print("RPi disconnected") log.addLog(from: "BT", message: "RPi disconnected") connected = false + startScanning() // try to reconnect } } diff --git a/Rear Rider/Rear Rider/ConfigurationOptions/OptionsView.swift b/Rear Rider/Rear Rider/ConfigurationOptions/OptionsView.swift index ab97222..7015c92 100644 --- a/Rear Rider/Rear Rider/ConfigurationOptions/OptionsView.swift +++ b/Rear Rider/Rear Rider/ConfigurationOptions/OptionsView.swift @@ -28,12 +28,19 @@ struct OptionsView: View { @State var confLightPattern: Int = 1 @State var confLightBrightness: Int = 1 @State var confLightColor: Color = .white + @State var confAlertToggle: Bool = true + @State var confVehiclesOnly: Bool = true + @State var confUnsafeDistance: Float = 17.0 // this index represents 900 in the array below + + var distance_text = ["50", "100", "150", "200", "250", "300", + "350", "400", "450", "500", "550", "600", + "650", "700", "750", "800", "850", "900", "950", "1000"] let audioFiles: [ConfigOptions.AudioFile] = ConfigOptions.AudioFile.allCases let lightPatterns: [ConfigOptions.LightPattern] = ConfigOptions.LightPattern.allCases let lightBrightness: [ConfigOptions.LightBrightness] = ConfigOptions.LightBrightness.allCases - let alert = RearRiderAlerts() + let alert = RearRiderAlerts.shared var body: some View { VStack { @@ -45,8 +52,14 @@ struct OptionsView: View { audioFile in Text(audioFile.description).tag(audioFile.rawValue) } }) + Toggle("Sound", isOn: $confAlertToggle.onChange(setAlert)) + Toggle("Vehicles Only", isOn: $confVehiclesOnly.onChange(setVehiclesOnly)) + HStack { + Slider(value: $confUnsafeDistance.onChange(setDistance), in: 0...19, step: 1) + Text("Distance: \(distance_text[Int(confUnsafeDistance)]) cm") + } } header: { - Text("Audio") + Text("LiDAR") } Section { Picker("Pattern", selection: $confLightPattern.onChange(setLights), content: { @@ -83,6 +96,9 @@ struct OptionsView: View { confLightBrightness = conf.lightBrightness confLightPattern = conf.lightPattern confLightColor = Color.fromRGBString(rgbString: conf.lightColor) + confAlertToggle = conf.alertToggle + confVehiclesOnly = conf.vehiclesOnly + confUnsafeDistance = Float(distance_text.lastIndex(of: String(conf.unsafeDistance)) ?? 0) } } @@ -100,6 +116,27 @@ struct OptionsView: View { } } + /// Enable alerts + /// - Parameter value: true to enable + func setAlert(to value: Bool) { + conf.alertToggle = confAlertToggle + saveConf() + } + + /// Set the unsafe distance + /// - Parameter value: the index of distance_text array + func setDistance(to value: Float) { + conf.unsafeDistance = Int(distance_text[Int(confUnsafeDistance)]) ?? 0 + saveConf() + } + + /// Alerts will be played only if the object detected is a vehicle + /// - Parameter value: true or false + func setVehiclesOnly(to value: Bool) { + conf.vehiclesOnly = confVehiclesOnly + saveConf() + } + /** * Sets the config object's light pattern to the new selection and saves it */ diff --git a/Rear Rider/Rear Rider/ConfigurationOptions/UserConfig.swift b/Rear Rider/Rear Rider/ConfigurationOptions/UserConfig.swift index 08a921a..395532d 100644 --- a/Rear Rider/Rear Rider/ConfigurationOptions/UserConfig.swift +++ b/Rear Rider/Rear Rider/ConfigurationOptions/UserConfig.swift @@ -16,6 +16,9 @@ struct ConfigData: Codable { let lightPattern: Int let lightBrightness: Int let lightColor: String + let alertToggle: Bool + let vehiclesOnly: Bool + let unsafeDistance: Int } /** @@ -24,12 +27,15 @@ struct ConfigData: Codable { enum ConfigOptions { enum AudioFile: String, CaseIterable, Equatable { case honk = "vehicleAlert" + case beep = "beep" case off = "" var description: String { switch self { case .honk: return "Honk" + case .beep: + return "Beep" case .off: return "None" } @@ -88,6 +94,9 @@ class UserConfig: ObservableObject { var lightPattern: Int = ConfigOptions.LightPattern.strobe.rawValue var lightColor: String = "rgb(255,255,255)" var lightBrightness: Int = 1 + var alertToggle: Bool = true + var vehiclesOnly: Bool = true + var unsafeDistance: Int = 900 var colorRGB: [CGFloat]? // use this for sending rgb values to RPi @@ -115,6 +124,14 @@ class UserConfig: ObservableObject { self.lightPattern = savedData.lightPattern self.lightBrightness = savedData.lightBrightness self.lightColor = savedData.lightColor + self.alertToggle = savedData.alertToggle + self.vehiclesOnly = savedData.vehiclesOnly + self.unsafeDistance = savedData.unsafeDistance + + try! RearRiderAlerts.shared.loadSoundFile(fileName: self.audioFile) + RearRiderAlerts.shared.alert_enabled = self.alertToggle + RearRiderAlerts.shared.vehicles_only = self.vehiclesOnly + RearRiderAlerts.shared.unsafe_distance = self.unsafeDistance print("LOADED: \(savedData)") log.addLog(from: "UserConfig", message: "Loaded config: \(savedData)") @@ -144,7 +161,10 @@ class UserConfig: ObservableObject { audioFile: audioFile, lightPattern: lightPattern, lightBrightness: lightBrightness, - lightColor: lightColor + lightColor: lightColor, + alertToggle: alertToggle, + vehiclesOnly: vehiclesOnly, + unsafeDistance: unsafeDistance ) let encoder = JSONEncoder() if let encodedData = try? encoder.encode(data) { @@ -153,6 +173,10 @@ class UserConfig: ObservableObject { log.addLog(from: "UserConfig", message: "Saved config: \(data)") } + RearRiderAlerts.shared.alert_enabled = self.alertToggle + RearRiderAlerts.shared.vehicles_only = self.vehiclesOnly + RearRiderAlerts.shared.unsafe_distance = self.unsafeDistance + /* Prepare data to be sent over BT * The format is as follows: * 1st byte - light pattern @@ -168,12 +192,14 @@ class UserConfig: ObservableObject { var red: UInt8 = UInt8(colorRGB[0] * 255) var green: UInt8 = UInt8(colorRGB[1] * 255) var blue: UInt8 = UInt8(colorRGB[2] * 255) + var distance: UInt16 = UInt16(self.unsafeDistance) bytes.append(withUnsafeBytes(of: &pat) { Data($0) }) bytes.append(withUnsafeBytes(of: &br) { Data($0) }) bytes.append(withUnsafeBytes(of: &red) { Data($0) }) bytes.append(withUnsafeBytes(of: &green) { Data($0) }) bytes.append(withUnsafeBytes(of: &blue) { Data($0) }) + bytes.append(withUnsafeBytes(of: &distance) { Data($0) }) bleManager.sendConfigToPi(data: bytes) } diff --git a/Rear Rider/Rear Rider/HomeView.swift b/Rear Rider/Rear Rider/HomeView.swift index 0c007da..7f0b3fd 100644 --- a/Rear Rider/Rear Rider/HomeView.swift +++ b/Rear Rider/Rear Rider/HomeView.swift @@ -25,13 +25,9 @@ struct HomeView: View { .foregroundColor(wifiManager.wifiOn ? .green : .red) } TabView { - RiderView() - .tabItem { - Image(systemName: "bicycle") - } CameraTestView() .tabItem { - Image(systemName: "camera") + Image(systemName: "bicycle") } OptionsView() .tabItem { diff --git a/Rear Rider/Rear Rider/ObjectIdentification/ImageIdentification.swift b/Rear Rider/Rear Rider/ObjectIdentification/ImageIdentification.swift index 0066631..78b6053 100644 --- a/Rear Rider/Rear Rider/ObjectIdentification/ImageIdentification.swift +++ b/Rear Rider/Rear Rider/ObjectIdentification/ImageIdentification.swift @@ -14,20 +14,16 @@ import Vision struct BoundingRect: Identifiable { var id: Int var rect: CGRect + var object: String + var confidence: Float } /// A class for detecting object using a Core ML model class ImageIdentification: ObservableObject { + static var shared = ImageIdentification() private var visionModel: VNCoreMLModel! - private var bndRects = [BoundingRect]() { - didSet { - DispatchQueue.main.async { - self.bndRectsCopy = self.bndRects // fix for Publishing from Background Thread purple warning - } - } - } - @Published var bndRectsCopy = [BoundingRect]() - var detected_objs = [String]() + var bndRects = [BoundingRect]() + private var locked: Bool = false init() { self.visionModel = initVisionModel() @@ -56,11 +52,11 @@ class ImageIdentification: ObservableObject { let detectedObject = result.labels.first?.identifier ?? "Nothing" let detectedObjectConfidence = result.labels.first?.confidence ?? 0 - self.detected_objs.append(detectedObject) - let temp = CGRect(x: result.boundingBox.origin.x, y: 1 - result.boundingBox.origin.y, width: result.boundingBox.width, height: result.boundingBox.height) // 389x288 is the size of the Image and ZStack views in CameraTestView with scaledToFit property - self.bndRects.append(BoundingRect(id: self.bndRects.count, rect: VNImageRectForNormalizedRect(temp, 389, 288))) + if !self.locked { + self.bndRects.append(BoundingRect(id: self.bndRects.count, rect: VNImageRectForNormalizedRect(temp, 389, 288), object: detectedObject, confidence: detectedObjectConfidence)) + } print("\(detectedObject) detected with \(detectedObjectConfidence) confidence") } @@ -97,6 +93,22 @@ class ImageIdentification: ObservableObject { /// Clear all bouding rects func clearBndRects() { self.bndRects.removeAll() - self.bndRectsCopy.removeAll() + } + + /// Check the labels of the detected object and compare it them with the allowed ones + /// - Returns: true or false + func checkForVehicles() -> Bool { + let labels: [String] = ["person", "bicycle", "car", "motorcycle", "bus", "truck"] + locked = true + + for obj in bndRects { + if labels.contains(obj.object) { + locked = false + return true + } + } + + locked = false + return false } } diff --git a/Rear Rider/Rear Rider/Rear_RiderApp.swift b/Rear Rider/Rear Rider/Rear_RiderApp.swift index 0689739..07efc67 100644 --- a/Rear Rider/Rear Rider/Rear_RiderApp.swift +++ b/Rear Rider/Rear Rider/Rear_RiderApp.swift @@ -26,9 +26,11 @@ struct Rear_RiderApp: App { @UIApplicationDelegateAdaptor(AppDelegate.self) var delegate @StateObject var viewRouter = ViewRouter() + @StateObject var mLModel = ImageIdentification.shared @StateObject var bleManager = BLEManager.shared @StateObject var conf = UserConfig() @StateObject var log = RearRiderLog.shared + @StateObject var alert = RearRiderAlerts.shared @StateObject var wifiManager = WifiManager.shared @StateObject var auth = AuthModel() @StateObject var db = FirestoreModel() @@ -36,6 +38,7 @@ struct Rear_RiderApp: App { var body: some Scene { WindowGroup { ViewController() + .environmentObject(mLModel) .environmentObject(bleManager) .environmentObject(log) .environmentObject(conf) @@ -43,6 +46,7 @@ struct Rear_RiderApp: App { .environmentObject(viewRouter) .environmentObject(auth) .environmentObject(db) + .environmentObject(alert) } } } diff --git a/Rear Rider/Rear Rider/RiderView.swift b/Rear Rider/Rear Rider/RiderView.swift index 526ff2b..0f97790 100644 --- a/Rear Rider/Rear Rider/RiderView.swift +++ b/Rear Rider/Rear Rider/RiderView.swift @@ -15,7 +15,7 @@ struct RiderView: View { var body: some View { VStack { ZStack { - ForEach (RearRiderAlerts.shared.mLModel.bndRectsCopy) { rect in + ForEach (ImageIdentification.shared.bndRects) { rect in Rectangle() .frame(width: rect.rect.width, height: rect.rect.height) .border(.yellow, width: 1) diff --git a/Rear Rider/Rear Rider/StreamingAndRecording/CameraTestView.swift b/Rear Rider/Rear Rider/StreamingAndRecording/CameraTestView.swift index 2f3f83b..c535cfc 100644 --- a/Rear Rider/Rear Rider/StreamingAndRecording/CameraTestView.swift +++ b/Rear Rider/Rear Rider/StreamingAndRecording/CameraTestView.swift @@ -10,19 +10,20 @@ import SwiftUI import AVKit struct CameraTestView: View { + @EnvironmentObject var mLModel: ImageIdentification @EnvironmentObject var bleManager: BLEManager @EnvironmentObject var wifiManager: WifiManager + @EnvironmentObject var alert: RearRiderAlerts @ObservedObject private var stream = MjpegStreamingController(url: "http://raspberrypi.local:8000/stream.mjpg") - private var mLModel = ImageIdentification() // declare a timer that will call a function every 0.3 seconds - private let timer = Timer.publish(every: 0.3, on: .main, in: .common) + private let timer = Timer.publish(every: 0.3, on: .main, in: .common).autoconnect() @State var timer_set = false var body: some View { VStack { ZStack { - ForEach (mLModel.bndRectsCopy) { rect in + ForEach (mLModel.bndRects) { rect in Rectangle() .frame(width: rect.rect.width, height: rect.rect.height) .border(.yellow, width: 1) @@ -42,35 +43,13 @@ struct CameraTestView: View { } .scaledToFit() - RecordingView() + Text(String(alert.distance) + " cm") - HStack { - Button(action: { - if wifiManager.wifiOn { - stream.play() - if !timer_set { - _ = timer.connect() - timer_set = true - } - } - }) { - Text("Play") - } - - Spacer() - - Button(action: { - mLModel.clearBndRects() - mLModel.detectObjects(image: stream.uiImage) - }) { - Text("Classify") - } - } + RecordingView() } .onAppear() { if (bleManager.connected) { bleManager.toggleNotifyCharacteristic(enabled: false) - wifiManager.turnWifiOn() } } .onReceive(timer) { time in @@ -79,18 +58,13 @@ struct CameraTestView: View { mLModel.detectObjects(image: stream.uiImage) } } - .onDisappear() { - if (wifiManager.wifiOn) { - wifiManager.turnWifiOff() - timer.connect().cancel() - timer_set = false - } - } } } struct CameraTest_Previews: PreviewProvider { static var previews: some View { CameraTestView().environmentObject(BLEManager()) + .environmentObject(ImageIdentification()) + .environmentObject(RearRiderAlerts()) } } diff --git a/Rear Rider/Rear Rider/StreamingAndRecording/MJpegStreamingKit.swift b/Rear Rider/Rear Rider/StreamingAndRecording/MJpegStreamingKit.swift index 10690c1..2f45e36 100644 --- a/Rear Rider/Rear Rider/StreamingAndRecording/MJpegStreamingKit.swift +++ b/Rear Rider/Rear Rider/StreamingAndRecording/MJpegStreamingKit.swift @@ -41,6 +41,7 @@ open class MjpegStreamingController: NSObject, URLSessionDataDelegate, Observabl super.init() self.session = Foundation.URLSession(configuration: URLSessionConfiguration.default, delegate: self, delegateQueue: nil) self.contentURL = URL(string: url) + self.play() } deinit { diff --git a/rear_rider_device/accelerometer_child_proc.py b/rear_rider_device/accelerometer_child_proc.py index 1d3fabf..25615dc 100644 --- a/rear_rider_device/accelerometer_child_proc.py +++ b/rear_rider_device/accelerometer_child_proc.py @@ -1,4 +1,5 @@ import asyncio +import threading from datetime import datetime from rear_rider_device.ipc.child_process import ChildProcess from typing import Deque @@ -18,16 +19,17 @@ def __init__(self, bt_server_proc: BluetoothServerChildProcess, buf_size: int = self.fps = fps self.ready = asyncio.Future() self.bt_server_proc = bt_server_proc + self._data_cond = threading.Condition() async def on_ready(self): self.start_time = datetime.now() self._print('AccelerometerChildProcess is ready.') - # try: - # await self.writeline('ready_ack') - try: + async def read_accelerometer(): await self.writeline('get_data') - except: - pass + with self._data_cond: + self._data_cond.wait(0.016) + return self.cyclic_buff.pop() + self.bt_server_proc.set_read_accelerometer_cb(read_accelerometer) self.ready.set_result(None) self._print('after_on_ready') @@ -59,13 +61,9 @@ async def on_data(self): float(component[1]), float(component[2]) ) - self.cyclic_buff.append(data) - await self.bt_server_proc.writeline( - 'set_data\n' - 'accelerometer\n' - '{},{},{}'.format(data[0], data[1], data[2])) - await asyncio.sleep(1.0/self.fps) - await self.writeline('get_data') + with self._data_cond: + self.cyclic_buff.append(data) + self._data_cond.notify_all() def _get_name(self) -> str: return 'AccelerometerChildProcess' diff --git a/rear_rider_device/bluetooth.py b/rear_rider_device/bluetooth.py index 317ab00..e08e10b 100644 --- a/rear_rider_device/bluetooth.py +++ b/rear_rider_device/bluetooth.py @@ -1,5 +1,6 @@ import sys import os +import threading PROJECT_ROOT = os.path.abspath(os.path.join( os.path.dirname(__file__), # This file should be in `rear_rider_device/` so we need to travel up one directory. @@ -10,12 +11,17 @@ import asyncio import concurrent.futures from concurrent.futures import Future, ThreadPoolExecutor -from typing import Any, Callable +from typing import Any, Callable, Union from rear_rider_device.ipc.parent_process import ParentProcess from rear_rider_device.rear_rider_bluetooth_server.src.services.characteristics.strobe_light import StrobeLight import rear_rider_device.rear_rider_bluetooth_server.src.main as bt_server_main -from rear_rider_device.rear_rider_bluetooth_server.src.services.hello_world import LedConfig +from rear_rider_device.rear_rider_bluetooth_server.src.services.hello_world import RearRiderConfig + +DATA_FUTURE_TIMEOUT = 0.016 +''' +The amount of seconds to wait for future data. +''' class BluetoothParentProcess(ParentProcess): rear_rider_bt: bt_server_main.RearRiderBluetooth @@ -27,6 +33,9 @@ class BluetoothParentProcess(ParentProcess): def __init__(self, bluetooth_ready: Future, create_strobe_light: Callable[[Any],StrobeLight]): self._bluetooth_ready = bluetooth_ready self.strobe_light = create_strobe_light(self) + self._accel_data_cond = threading.Condition() + self._accel_data: Union[None, tuple[float, float, float]] = None + async def pre_ready(self): print('pre_ready') @@ -36,9 +45,11 @@ async def pre_ready(self): # self._bluetooth_ready.result() # FOR DEBUGGING ONLY # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #################################################### - def on_led_config(cfg: LedConfig): + def on_rear_rider_config(cfg: RearRiderConfig): self.writeline(f'led_config\n{cfg.pattern} {cfg.brightness} {cfg.color[0]} {cfg.color[1]} {cfg.color[2]}') - self.rear_rider_bt.hello_world_svc.config_chr.set_on_led_config(on_led_config) + self.writeline(f'lidar_config\n{cfg.lidar_unsafe_distance}') + self.rear_rider_bt.hello_world_svc.config_chr.set_on_config(on_rear_rider_config) + self.rear_rider_bt.sensors_svc.accelerometer_characteristic.set_read_accelerometer_cb(self.read_accelerometer) self.writeline('bluetooth_is_ready') async def pre_loop(self): @@ -60,12 +71,15 @@ async def on_set_data(self): data_type_line = await self.readline() if data_type_line == 'accelerometer': - data = await self.readline() # TODO: Add critical section guard here # self.writeline('set_data_ack') - nums = data.split(',') - self.rear_rider_bt.sensors_svc.accelerometer_characteristic.vector = ( - float(nums[0]),float(nums[1]),float(nums[2])) + with self._accel_data_cond: + self._accel_data = _parse_acceleration_data(await self.readline()) + self._accel_data_cond.notify_all() + elif data_type_line == 'lidar': + data = await self.readline() + self.rear_rider_bt.hello_world_svc.lidar_chr.value = data + self.rear_rider_bt.hello_world_svc.lidar_chr.check_object_in_range() else: # TODO: Add critical section guard here # self.writeline('set_data_ack') @@ -91,8 +105,24 @@ def is_strobe_on(self): def discoverable_changed(self, value: str): timeout = self.rear_rider_bt.get_discoverable_timeout() self.writeline(f'discoverable\n{value} {timeout}') - + def read_accelerometer(self) -> tuple[float, float, float]: + ''' + This action could timeout. + ''' + with self._accel_data_cond: + self.writeline('read_accelerometer') + try: + self._accel_data_cond.wait(DATA_FUTURE_TIMEOUT) + if self._accel_data is None: + raise Exception('Acceleration data was None') + return self._accel_data + finally: + self._accel_data = None + +def _parse_acceleration_data(line: str): + nums = line.split(',') + return (float(nums[0]),float(nums[1]),float(nums[2])) if __name__ == '__main__': # TODO: Synchronize writes to stdout using `with` keyword: @@ -122,15 +152,8 @@ def bluetooth_is_ready(rear_rider_bt: bt_server_main.RearRiderBluetooth): proc.rear_rider_bt = rear_rider_bt proc._bluetooth_ready.set_result(None) - x=0 - y=0 - def on_read(): - x = x + 1 - y = y + 1 - return '{},{}'.format(x, y) bt_server_main.main(print, on_ready=bluetooth_is_ready, - on_read=on_read, strobe_light=proc.strobe_light, ) diff --git a/rear_rider_device/bluetooth_server_child_proc.py b/rear_rider_device/bluetooth_server_child_proc.py index 8947442..3c08522 100644 --- a/rear_rider_device/bluetooth_server_child_proc.py +++ b/rear_rider_device/bluetooth_server_child_proc.py @@ -1,26 +1,24 @@ -import asyncio -import concurrent.futures -from rear_rider_device.ipc.child_process import ChildProcess -from rear_rider_device.ipc.parent_process import ParentProcess - import os - +from typing import Awaitable, Callable, Union +from rear_rider_device.ipc.child_process import ChildProcess from rear_rider_device.leds_child_proc import LedsChildProcess dir_path = os.path.dirname(os.path.realpath(__file__)) - - class BluetoothServerChildProcess(ChildProcess): + ''' + This class handles the messages received from the bluetooth server child process. + ''' def __init__(self, leds_child_process: LedsChildProcess): - super().__init__("python {}/bluetooth.py".format(dir_path)) - self._leds_child_process = leds_child_process - + super().__init__(f'python {dir_path}/bluetooth.py') + self._leds_child_process: LedsChildProcess = leds_child_process + self._read_accelerometer_cb: Union[None, Callable[[], Awaitable[tuple[float, float, float]]]] = None + self._lidar_config_cb: Union[None, Callable[[int], None]] = None + def _get_name(self) -> str: return 'BluetoothServerChildProcess' - def _print_header(self, message: str): - return super()._print('==== Bluetooth Process ====\n{}'.format(message)) + return super()._print(f'==== Bluetooth Process ====\n{message}') async def on_wait_ready(self): while True: @@ -36,14 +34,27 @@ def on_done(self): async def on_sensor_data_stream_ready(self): self._print_header('sensor_data_stream begin') - self._print_header('sensor_data_stream_end') - + async def on_set_data_ack(self): self._print('on_set_data_ack') async def on_read_accelerometer(self): self._print('on_read_accelerometer') + accel = await self._read_accelerometer() + await self.writeline( + 'set_data\n' + 'accelerometer\n' + f'{accel[0]},{accel[1]},{accel[2]}') + + async def _read_accelerometer(self): + read_accelerometer_cb = self._read_accelerometer_cb + if read_accelerometer_cb is None: + raise Exception('The read_accelerometer_cb is None') + return await read_accelerometer_cb() + + def set_read_accelerometer_cb(self, callback: Callable[[], Awaitable[tuple[float, float, float]]]): + self._read_accelerometer_cb = callback ############# # LED STUFF # @@ -64,30 +75,28 @@ async def on_led_strobe_off(self): """ self._print('on_led_strobe_off') await self._leds_child_process.led_strobe_off() - + async def is_strobe_on(self): self._print('is_strobe_on') - + async def is_strobe_on_response(self, strobe_on: bool): await self.writeline( 'strobe_on\n' - '{}'.format(strobe_on)) - # await self._wait_ack('strobe_on') - + f'{strobe_on}') + async def on_no_on_handler(self): line = await self.readline() - self._print('on no on handler: {}'.format(line)) + self._print(f'on no on handler: {line}') async def on_help(self): while True: line = await self.readline() if line == '== help_string_end ==': break - pass def no_on_handler(self, on_command, err): - self._print('no on handler s: {}\n,{}'.format(on_command, err)) - + self._print(f'no on handler s: {on_command}\n,{err}') + async def on_discoverable(self): """ The handler for when bluetooth changes its discoverability to other devices. @@ -99,27 +108,30 @@ async def on_discoverable(self): await self._leds_child_process.set_discoverable_effect(True) elif discoverable == '0': await self._leds_child_process.set_discoverable_effect(False) - + async def on_led_config(self): line = await self.readline() self._print(f'led_config: {line}') await self._leds_child_process.add_led_effect(line) + async def on_lidar_config(self): + ''' + The handler for the configuration data received from the bluetooth side. - -if __name__ == '__main__': - leds_proc = LedsChildProcess() - proc = BluetoothServerChildProcess(leds_child_process=leds_proc) - # parent = TestParent(leds_proc) - processes: list = [ - proc, - leds_proc - ] - futures = [] - with concurrent.futures.ThreadPoolExecutor() as executor: - # We create a thread for each to-be-executed child process - # so that asyncio manages one child process per thread. - for process in processes: - futures.append(executor.submit(asyncio.run, process.begin())) - print(len(futures)) - concurrent.futures.wait(futures) + This handler function will use the callback set in `self.set_lidar_config_cb(...)` + ''' + line = await self.readline() + self._print(f'lidar_config: {line}') + self.__set_lidar_config(int(line)) + + def __set_lidar_config(self, dist_cfg: int): + if self._lidar_config_cb is None: + return + self._lidar_config_cb(dist_cfg) + + def set_lidar_config_cb(self, callback: Callable[[int], None]): + ''' + Set the callback `BluetoothServerChildProcess` will call when a new configuration value for + the lidar sensor is received. + ''' + self._lidar_config_cb = callback \ No newline at end of file diff --git a/rear_rider_device/camera_proc.py b/rear_rider_device/camera_proc.py index 17cdec0..8fc7964 100644 --- a/rear_rider_device/camera_proc.py +++ b/rear_rider_device/camera_proc.py @@ -1,9 +1,18 @@ +import sys +import os +PROJECT_ROOT = os.path.abspath(os.path.join( + os.path.dirname(__file__), + # This file should be in `rear_rider_device/` so we need to travel up one directory. + f'{os.pardir}') +) +sys.path.append(PROJECT_ROOT) + import asyncio from pkgutil import get_data import readline from sys import stdout -from ipc.parent_process import ParentProcess -import rear_rider_sensors.camera as camera +from rear_rider_device.ipc.parent_process import ParentProcess +import rear_rider_device.rear_rider_sensors.camera as camera import os from datetime import datetime from datetime import date diff --git a/rear_rider_device/lidar_child_proc.py b/rear_rider_device/lidar_child_proc.py index 331aab6..f7587cd 100644 --- a/rear_rider_device/lidar_child_proc.py +++ b/rear_rider_device/lidar_child_proc.py @@ -2,24 +2,35 @@ from concurrent.futures import ThreadPoolExecutor import concurrent.futures from datetime import datetime +from rear_rider_device.bluetooth_server_child_proc import BluetoothServerChildProcess from rear_rider_device.ipc.child_process import ChildProcess from rear_rider_device.leds_child_proc import LedsChildProcess import os dir_path = os.path.dirname(os.path.realpath(__file__)) +DEFAULT_UNSAFE_DISTANCE = 500 + class LidarChildProcess(ChildProcess): lidar_distance = 0 # default unit is cm signal_strength = 0 # signal unreliable under 100 - def __init__(self, led_child_proc: LedsChildProcess): + def __init__(self, led_child_proc: LedsChildProcess, + bt_child_proc: BluetoothServerChildProcess): """ Default `buf_size` of 64 frame datapoints at 60 `fps`. """ super().__init__('python {}/lidar_proc.py'.format(dir_path)) self.ready = asyncio.Future() self.led_child_proc = led_child_proc + self.in_range = False + self.bt_child_proc = bt_child_proc + self._unsafe_distance = DEFAULT_UNSAFE_DISTANCE + def set_lidar_config(dist_cfg: int): + self._unsafe_distance = dist_cfg + self._print(f'Distance config: {self._unsafe_distance} cm') + self.bt_child_proc.set_lidar_config_cb(set_lidar_config) async def on_ready(self): @@ -38,10 +49,11 @@ async def on_data(self): lidar_data = (await self.readline()).split(' ') lidar_distance = lidar_data[0] signal_strength = lidar_data[1] - self._print('Lidar_distance:{}\n\tSignal_strength:{}\n'.format(lidar_distance, signal_strength)) + #self._print('Lidar_distance:{}\n\tSignal_strength:{}\n'.format(lidar_distance, signal_strength)) - unsafe_distance = 50 - if int(lidar_distance) < unsafe_distance: + unsafe_distance = self._unsafe_distance + if int(lidar_distance) <= unsafe_distance: + await self.bt_child_proc.writeline('set_data\nlidar\n{}'.format(lidar_distance)) await self.led_child_proc.led_strobe_on() else: await self.led_child_proc.led_strobe_off() diff --git a/rear_rider_device/main.py b/rear_rider_device/main.py old mode 100644 new mode 100755 index 012981d..9e9de7b --- a/rear_rider_device/main.py +++ b/rear_rider_device/main.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import os import sys PROJECT_ROOT = os.path.abspath(os.path.join( @@ -23,14 +24,15 @@ def main(): leds_child_process=leds_child_proc) accelerometer_proc = AccelerometerChildProcess(buf_size=32, fps=1, bt_server_proc=bt_server_process) - lidar_child_proc = LidarChildProcess(led_child_proc=leds_child_proc) - # camera_proc = CameraChildProcess(bt_server_proc=bt_server_process) + lidar_child_proc = LidarChildProcess(led_child_proc=leds_child_proc, + bt_child_proc=bt_server_process) + camera_proc = CameraChildProcess() child_processes: list[Process] = [ leds_child_proc, accelerometer_proc, bt_server_process, - lidar_child_proc - # camera_proc + lidar_child_proc, + camera_proc ] futures = [] with concurrent.futures.ThreadPoolExecutor() as executor: diff --git a/rear_rider_device/rear_rider_bluetooth_server/src/bluetooth_device.py b/rear_rider_device/rear_rider_bluetooth_server/src/bluetooth_device.py index 1a0509f..9bdb9ec 100644 --- a/rear_rider_device/rear_rider_bluetooth_server/src/bluetooth_device.py +++ b/rear_rider_device/rear_rider_bluetooth_server/src/bluetooth_device.py @@ -10,15 +10,26 @@ class BluetoothDevice: Represents a bluetooth device. """ def __init__(self, bus: dbus.Bus, device_path: str): - self._device = dbus.Interface( + self._props = dbus.Interface( bus.get_object(BLUEZ_SERVICE_NAME, device_path), DBUS_PROPS_IFACE) + self._device = dbus.Interface( + bus.get_object(BLUEZ_SERVICE_NAME, device_path), + BLUEZ_DEVICE_1) + + def get_address(self): + ''' + Return the bluetooth MAC address of this device. + ''' + return str(self._props.Get(BLUEZ_DEVICE_1, 'Address')) - def address(self): - return str(self._device.Get('org.bluez.Device1', 'Address')) - - def get_object_path(self): - return str(self._device.object_path) - def disconnect(self): + ''' + Disconnect this device. + ''' self._device.Disconnect() + + def __str__(self) -> str: + return ( + f'Address: {self.get_address()}' + ) diff --git a/rear_rider_device/rear_rider_bluetooth_server/src/main.py b/rear_rider_device/rear_rider_bluetooth_server/src/main.py index 1292a77..e4f3162 100755 --- a/rear_rider_device/rear_rider_bluetooth_server/src/main.py +++ b/rear_rider_device/rear_rider_bluetooth_server/src/main.py @@ -106,9 +106,11 @@ def _set_pairable(self, value: bool): self._adapter_props.Set('org.bluez.Adapter1', 'Pairable', dbus.Boolean(value)) def _on_device_connection_change(self, connected: bool, device_path: str): + device = BluetoothDevice(self._bus, device_path) if connected: - device = BluetoothDevice(self._bus, device_path) if self._connected_device is not None: + if device.get_address() == self._connected_device.get_address(): + return device.disconnect() raise Exception('We only want one device at a time to be connected.') # Disable pairing since we only want one device at a time to be connected. @@ -119,11 +121,11 @@ def _on_device_connection_change(self, connected: bool, device_path: str): return # connect == False, therefore this device was just disconnected. if (self._connected_device is not None and - self._connected_device.get_object_path() != device_path): + self._connected_device.get_address() != device.get_address()): # Since we only expect one device to be connected, we do not expect to reach this. raise Exception( 'Expected _connected_device to be not None and the object paths to be the same.\n' - f'{self._connected_device.get_object_path()} {device_path}') + f'{self._connected_device.get_address()} {device.get_address()}') self._connected_device = None self._set_pairable(True) self.set_discoverable('1') @@ -132,15 +134,13 @@ def has_connected_device(self): return self._connected_device is not None -def main(print, on_ready: Union[None, Callable[[RearRiderBluetooth], None]], on_read: Callable[[], str], - strobe_light: StrobeLight): +def main(print, on_ready: Union[None, Callable[[RearRiderBluetooth], None]], strobe_light: StrobeLight): """ """ # First check all the variables are not none in order to ensure the main is valid. try: assert(print != None) assert(on_ready != None) - assert(on_read != None) assert(strobe_light != None) except AssertionError: print('Condition for main function not met!') @@ -165,7 +165,7 @@ def main(print, on_ready: Union[None, Callable[[RearRiderBluetooth], None]], on_ ad_manager = get_object_interface(LE_ADVERTISING_MANAGER_IFACE) - app = RearRiderApplication(bus, read_data=on_read, + app = RearRiderApplication(bus, strobe_light=strobe_light ) diff --git a/rear_rider_device/rear_rider_bluetooth_server/src/rearrider_app.py b/rear_rider_device/rear_rider_bluetooth_server/src/rearrider_app.py index f3f6e3b..44c0a61 100644 --- a/rear_rider_device/rear_rider_bluetooth_server/src/rearrider_app.py +++ b/rear_rider_device/rear_rider_bluetooth_server/src/rearrider_app.py @@ -9,13 +9,13 @@ class RearRiderApplication(dbus.service.Object): org.bluez.GattApplication1 interface implementation """ services: list[Service] - def __init__(self, bus, read_data, strobe_light: StrobeLight): + def __init__(self, bus, strobe_light: StrobeLight): self.path = '/' self.services = [] dbus.service.Object.__init__(self, bus, self.path) hello_world_service = HelloWorldService(bus, 0) - sensors_service = SensorsService(bus, 1, read_data) + sensors_service = SensorsService(bus, 1) actuators_service = ActuatorsService(bus, 2, strobe_light) self.add_service(hello_world_service) diff --git a/rear_rider_device/rear_rider_bluetooth_server/src/services/characteristics/accelerometer_characteristic.py b/rear_rider_device/rear_rider_bluetooth_server/src/services/characteristics/accelerometer_characteristic.py new file mode 100644 index 0000000..2adcc06 --- /dev/null +++ b/rear_rider_device/rear_rider_bluetooth_server/src/services/characteristics/accelerometer_characteristic.py @@ -0,0 +1,111 @@ +from typing import Callable, Union +import dbus +from rear_rider_device.rear_rider_bluetooth_server.src.bluez.example_gatt_server import \ + GATT_CHRC_IFACE, Characteristic, Descriptor, FailedException +from rear_rider_device.utils.threaded_repeat import ThreadedRepeat + + +class AccelerometerCharacteristic(Characteristic): + UUID = '6df2bcb1-a775-42ea-91c2-14d22c1c8f48' + def __init__(self, bus, index, service): + super().__init__(bus, index, self.UUID, + ['read', 'write', 'notify'], service) + self._read_accelerometer_cb: Union[None, Callable[[], tuple[float, float, float]]] = None + self._accel: tuple[float, float, float] = (0.0,0.0,0.0) + self._notifier: Union[None, ThreadedRepeat] = None + self._accelerometer_notify_interval = AccelerometerNotifyIntervalDescriptor( + bus, 0, self + ) + def on_accel_notify_interval_change(interval_ms: int): + try: + self._notifier.update_interval(interval_seconds=interval_ms/1000.0) # type: ignore + except: + # try block may throw if _notifier is None. Since it may be set to None by another + # thread let's avoid using a mutex lock by just catching and passing the exception. + pass + self._accelerometer_notify_interval.set_on_change_cb(on_accel_notify_interval_change) + self.descriptors.append(self._accelerometer_notify_interval) + + + def StartNotify(self): + if self._notifier is not None: + return + def notify(): + self.PropertiesChanged(GATT_CHRC_IFACE, { + 'Value': self._read_accelerometer() + }, []) + self._notifier = ThreadedRepeat(self._accelerometer_notify_interval.get_interval()/1000, + notify) + self._notifier.start() + + def StopNotify(self): + if self._notifier is None: + return + self._notifier.cancel() + self._notifier = None + + def ReadValue(self, options): + return self._read_accelerometer() + + def _read_accelerometer(self): + ''' + Raises an 'org.bluez.Error.Failed' error if there was no callback assigned via + `self.set_read_accelerometer_cb(...)`. + + The value is read, `self.vector` is set to that value, then it is formatted as a dbus byte + array type (which is encoded as a utf8 string) and returned. + ''' + if self._read_accelerometer_cb is None: + raise FailedException('The callback to read the accelerometer data is None.') + self._accel = self._read_accelerometer_cb() + return _dbus_format_accel_vector(self._accel) + + def set_read_accelerometer_cb(self, callback: Callable[[],tuple[float, float, float]]): + ''' + Sets the callback function for when the bluetooth characteristic is being accessed for + reads or for acquire-notifications. + ''' + self._read_accelerometer_cb = callback + +def _dbus_format_accel_vector(vector: tuple[float, float, float]): + ''' + Format a 3 vector tuple as a dbus ByteArray type (a utf 8 string). + ''' + return dbus.ByteArray(f'{vector[0]},{vector[1]},{vector[2]}'.encode('utf8')) + +class AccelerometerNotifyIntervalDescriptor(Descriptor): + UUID = 'e01f9dab-64c6-429f-8e89-3f185392c327' + + def __init__(self, bus, index, characteristic, interval_ms=1000): + super().__init__(bus, index, self.UUID, + ['read', 'write'], + characteristic) + self._interval_ms = interval_ms + ''' + In milliseconds. + ''' + self._on_change_cb: Union[None, Callable[[int], None]] = None + + def ReadValue(self, options): + try: + return dbus.ByteArray(dbus.UInt16(self._interval_ms).to_bytes(2, 'big')) + except Exception as e: + print(e) + + def WriteValue(self, value, options): + """ + Expects bytes representing a uint16 in big-endian ordering. + """ + interval = int.from_bytes(value, 'big') + self._interval_ms = interval + if self._on_change_cb is not None: + self._on_change_cb(interval) + + def set_on_change_cb(self, callback: Callable[[int], None]): + self._on_change_cb = callback + + def get_interval(self): + ''' + Get the interval in milliseconds. + ''' + return self._interval_ms diff --git a/rear_rider_device/rear_rider_bluetooth_server/src/services/hello_world.py b/rear_rider_device/rear_rider_bluetooth_server/src/services/hello_world.py index a4caa38..aec2de2 100644 --- a/rear_rider_device/rear_rider_bluetooth_server/src/services/hello_world.py +++ b/rear_rider_device/rear_rider_bluetooth_server/src/services/hello_world.py @@ -39,15 +39,18 @@ def __init__(self, bus, index): config_chr = ConfigCharacteristic(bus, 2, self) wifi_chr = WifiCharacteristic(bus, 3, self) pic_chr = PictureCharacteristic(bus, 4, self) + lidar_chr = LiDARCharacteristic(bus, 5, self) self.add_characteristic(reverse_text_chr) self.add_characteristic(append_counter_chr) self.add_characteristic(config_chr) self.add_characteristic(wifi_chr) self.add_characteristic(pic_chr) + self.add_characteristic(lidar_chr) self.reverse_text_chr = reverse_text_chr self.config_chr = config_chr + self.lidar_chr = lidar_chr class ReverseTextCharacteristic(Characteristic): """ @@ -116,7 +119,7 @@ def _increment_counter(self): return self.notifying @dataclass -class LedConfig: +class RearRiderConfig: pattern: int = 0 """ 0 - no pattern @@ -131,13 +134,20 @@ class LedConfig: """ (r, g, b) """ + lidar_unsafe_distance: int = 1 + """ + Distance represented in centimeters. + """ def to_bytes(self): - return [self.pattern, self.brightness, *self.color] + return [self.pattern, self.brightness, *self.color, + *self.lidar_unsafe_distance.to_bytes(length=2, byteorder='little')] class ConfigCharacteristic(Characteristic): """ Configure the LED lights. + + TODO: Move this into seperate file. """ TEST_CHRC_UUID = '501beabd-3f66-4cca-ba7a-0fbf4f81870c' @@ -147,12 +157,12 @@ def __init__(self, bus, index, service): self.TEST_CHRC_UUID, ['write'], service) - self.value: LedConfig - self._on_led_config: Union[None, Callable[[LedConfig], None]] = None + self.value: RearRiderConfig + self._on_led_config: Union[None, Callable[[RearRiderConfig], None]] = None self._config_characteristic__init__() def _config_characteristic__init__(self): - self.value = LedConfig() + self.value = RearRiderConfig() def WriteValue(self, value, options): pattern = int(value[0]) @@ -160,12 +170,13 @@ def WriteValue(self, value, options): r = int(value[2]) g = int(value[3]) b = int(value[4]) + lidar_unsafe_distance = int.from_bytes([value[5], value[6]], byteorder='little') # print(f'ConfigCharacteristic Write: {pattern} {brightness} {r} {g} {b}') - self.value = LedConfig(pattern, brightness, (r, g, b)) + self.value = RearRiderConfig(pattern, brightness, (r, g, b), lidar_unsafe_distance) if self._on_led_config is not None: self._on_led_config(self.value) - def set_on_led_config(self, callback: Callable[[LedConfig], None]): + def set_on_config(self, callback: Callable[[RearRiderConfig], None]): self._on_led_config = callback @@ -272,3 +283,34 @@ def ReadValue(self, options): def WriteValue(self, value, options): self.value = int(value[0]) + +class LiDARCharacteristic(Characteristic): + """ + Notifies the iOS app when an object is detected by the LiDAR sensor and sends + the distance. + """ + UUID = '92cb916f-d996-4f30-8cba-cf3ab8aede56' + value: dbus.ByteArray + def __init__(self, bus, index, service): + Characteristic.__init__( + self, bus, index, + self.UUID, + ['notify'], + service) + self.notifying = False + self.value = 0 + + def StartNotify(self): + if self.notifying: + return + self.notifying = True + + def StopNotify(self): + if not self.notifying: + return + self.notifying = False + + def check_object_in_range(self): + value = dbus.ByteArray(self.value.encode('utf8')) + self.PropertiesChanged(GATT_CHRC_IFACE, { 'Value': value }, []) + return self.notifying diff --git a/rear_rider_device/rear_rider_bluetooth_server/src/services/sensors.py b/rear_rider_device/rear_rider_bluetooth_server/src/services/sensors.py index 66c6781..b5c88b2 100644 --- a/rear_rider_device/rear_rider_bluetooth_server/src/services/sensors.py +++ b/rear_rider_device/rear_rider_bluetooth_server/src/services/sensors.py @@ -1,6 +1,7 @@ -from sys import stdin, stdout from typing import Callable -from rear_rider_device.rear_rider_bluetooth_server.src.bluez.example_gatt_server import dbus, GATT_CHRC_IFACE, Characteristic, Service, GObject +from rear_rider_device.rear_rider_bluetooth_server.src.bluez.example_gatt_server import Service +from rear_rider_device.rear_rider_bluetooth_server.src.services.characteristics.\ + accelerometer_characteristic import AccelerometerCharacteristic class SensorsService(Service): """ @@ -8,64 +9,13 @@ class SensorsService(Service): """ SENSORS_SVC_UUID = 'f0135e21-ad28-46e3-af7a-6e0829ab4c4a' - def __init__(self, bus, index, read_data: Callable[[], str]): + def __init__(self, bus, index): Service.__init__(self, bus, index, self.SENSORS_SVC_UUID, True) accelerometer_characteristic = AccelerometerCharacteristic( - bus, 0, self, read_data=read_data) + bus, 0, self) # self.add_characteristic(CameraFeedCharacteristic(bus, 0, self)) self.add_characteristic(accelerometer_characteristic) # self.add_characteristic(RadarCharacteristic(bus, 2, self)) self.accelerometer_characteristic = accelerometer_characteristic - -class AccelerometerCharacteristic(Characteristic): - UUID = '6df2bcb1-a775-42ea-91c2-14d22c1c8f48' - def __init__(self, bus, index, service, read_data: Callable[[], str]): - super().__init__(bus, index, self.UUID, - ['read', 'write', 'notify'], service) - self.notifying = False - self.read_data = read_data - self.value = [] - self.vector: tuple[float, float, float] = (0.0,0.0,0.0) - - def StartNotify(self): - if self.notifying: - return - self.notifying = True - def notify(): - value = self._get_vector_data() - self.PropertiesChanged(GATT_CHRC_IFACE, { 'Value': value }, []) - print('DATA: {}'.format(value)) - return self.notifying - - GObject.timeout_add(1000, notify) - - def StopNotify(self): - if not self.notifying: - return - self.notifying = False - - def ReadValue(self, options): - # TODO: Move this into a callback function. - line = 'accelerometer' - if line != 'accelerometer': - return - stdout.write('read_accelerometer\n') - stdout.flush() - nums = self.vector - - data = self._get_vector_data() - print(data) - return data - - def _get_vector_data(self): - """ - As a byte string in ut8. - """ - nums = self.vector - return dbus.ByteArray('{},{},{}'.format( - nums[0], nums[1], nums[2] - ).encode('utf8')) - - diff --git a/rear_rider_device/utils/threaded_repeat.py b/rear_rider_device/utils/threaded_repeat.py new file mode 100644 index 0000000..a5c4a5f --- /dev/null +++ b/rear_rider_device/utils/threaded_repeat.py @@ -0,0 +1,19 @@ +import threading +from typing import Callable + +class ThreadedRepeat(threading.Timer): + ''' + Repeat a function at a set interval. + ''' + def __init__(self, interval_seconds: float, function: Callable[[], None]): + super().__init__(interval_seconds, function) + + def run(self): + while not self.finished.wait(self.interval): + self.function() + + def update_interval(self, interval_seconds: float): + ''' + The interval in seconds. + ''' + self.interval = interval_seconds diff --git a/rearrider_180x180.png b/rearrider_180x180.png new file mode 100644 index 0000000..b69f056 Binary files /dev/null and b/rearrider_180x180.png differ diff --git a/test/test_rear_rider_device/test_bluetooth/test_config_characteristic.py b/test/test_rear_rider_device/test_bluetooth/test_config_characteristic.py index b790f9b..31db624 100644 --- a/test/test_rear_rider_device/test_bluetooth/test_config_characteristic.py +++ b/test/test_rear_rider_device/test_bluetooth/test_config_characteristic.py @@ -1,4 +1,4 @@ -from rear_rider_device.rear_rider_bluetooth_server.src.services.hello_world import ConfigCharacteristic, LedConfig +from rear_rider_device.rear_rider_bluetooth_server.src.services.hello_world import ConfigCharacteristic, RearRiderConfig import dbus import unittest @@ -12,13 +12,14 @@ def __init__(self): #pylint: disable=super-init-not-called class TestConfigCharacteristic(unittest.TestCase): def test_set_on_led_config(self): callback_called = False - config_val = [0x01, 0x01, 0xFF, 0xFF, 0xFF] - def callback(new_led_config: LedConfig): + little_endian_int = [0x00, 0x02] + config_val = [0x01, 0x01, 0xFF, 0xFF, 0xFF, *little_endian_int] + def callback(new_led_config: RearRiderConfig): nonlocal callback_called callback_called = True self.assertEqual(new_led_config.to_bytes(), config_val) test_cfg_chara = StubbedConfigCharacteristic() - test_cfg_chara.set_on_led_config(callback=callback) + test_cfg_chara.set_on_config(callback=callback) test_cfg_chara.WriteValue(dbus.ByteArray(config_val), None) self.assertTrue(callback_called, 'The on_led_config callback was not called.') diff --git a/test/test_rear_rider_device/test_ipc/child_processes/test_bluetooth_server_child_proc.py b/test/test_rear_rider_device/test_ipc/child_processes/test_bluetooth_server_child_proc.py new file mode 100644 index 0000000..eb25f88 --- /dev/null +++ b/test/test_rear_rider_device/test_ipc/child_processes/test_bluetooth_server_child_proc.py @@ -0,0 +1,33 @@ +""" +Refactored from `rear_rider_device/bluetooth_server_child_proces.py`. +""" +import asyncio +import concurrent.futures +import sys +import os +PROJECT_ROOT = os.path.abspath(os.path.join( + os.path.dirname(__file__), + # This file should be in `test/rear_rider_device/test_ipc/child_processes` so we need to travel + # up 3 directories. + f'{os.pardir}/../../..') +) +sys.path.append(PROJECT_ROOT) +from rear_rider_device.bluetooth_server_child_proc import BluetoothServerChildProcess +from rear_rider_device.leds_child_proc import LedsChildProcess + +if __name__ == '__main__': + leds_proc = LedsChildProcess() + proc = BluetoothServerChildProcess(leds_child_process=leds_proc) + # parent = TestParent(leds_proc) + processes: list = [ + proc, + leds_proc + ] + futures = [] + with concurrent.futures.ThreadPoolExecutor() as executor: + # We create a thread for each to-be-executed child process + # so that asyncio manages one child process per thread. + for process in processes: + futures.append(executor.submit(asyncio.run, process.begin())) + print(len(futures)) + concurrent.futures.wait(futures)