Skip to content

Commit

Permalink
Use Swift Concurrency instead of Dispatch (#77)
Browse files Browse the repository at this point in the history 
Also moved json loading inside of task group to speed things up a bit
  • Loading branch information
@adam-fowler
adam-fowler committed Apr 6, 2024
1 parent 4321cde commit d27d783
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 62 deletions.
3 changes: 2 additions & 1 deletion Package.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ import PackageDescription

let package = Package(
name: "soto-codegenerator",
platforms: [.macOS(.v10_15)],
products: [
.executable(name: "SotoCodeGenerator", targets: ["SotoCodeGenerator"]),
.plugin(name: "SotoCodeGeneratorPlugin", targets: ["SotoCodeGeneratorPlugin"]),
],
dependencies: [
.package(url: "https://github.com/soto-project/soto-smithy.git", from: "0.3.1"),
.package(url: "https://github.com/soto-project/soto-smithy.git", from: "0.3.7"),
.package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.0"),
.package(url: "https://github.com/hummingbird-project/hummingbird-mustache.git", from: "1.0.3"),
.package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"),
Expand Down
9 changes: 4 additions & 5 deletions Sources/SotoCodeGenerator/main.swift → ...SotoCodeGenerator/SotoCodeGenerator.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ import Foundation
import Logging
import SotoCodeGeneratorLib

struct Command: ParsableCommand, SotoCodeGenCommand {
@main
struct Command: AsyncParsableCommand, SotoCodeGenCommand {
@Option(name: .long, help: "Folder to output service files to")
var outputFolder: String

Expand Down Expand Up @@ -63,9 +64,7 @@ struct Command: ParsableCommand, SotoCodeGenCommand {
static var defaultInputFolder: String { return "\(rootPath)/aws/models" }
static var defaultEndpoints: String { return "\(rootPath)/aws/endpoints.json" }

func run() throws {
try SotoCodeGen(command: self).generate()
func run() async throws {
try await SotoCodeGen(command: self).generate()
}
}

Command.main()
125 changes: 69 additions & 56 deletions Sources/SotoCodeGeneratorLib/SotoCodeGen.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,60 +48,65 @@ public struct SotoCodeGen {
var logger = Logging.Logger(label: "SotoCodeGenerator")
logger.logLevel = self.command.logLevel.map { Logging.Logger.Level(rawValue: $0) ?? .info } ?? .info
self.logger = logger
Smithy.registerAWSTraits()
Smithy.registerTraitTypes(
SotoInputShapeTrait.self,
SotoOutputShapeTrait.self
)
}

public func generate() throws {
public func generate() async throws {
let startTime = Date()

// load JSON
let config = try loadConfigFile()
let endpoints = try loadEndpointJSON()
let models: [String: SotoSmithy.Model]
let modelFiles: [String]
if self.command.smithy {
models = try self.loadSmithy()
modelFiles = self.getSmithyFiles()
} else {
models = try self.loadModelJSON()
modelFiles = self.getModelFiles()
}
let group = DispatchGroup()

models.forEach { model in
group.enter()

DispatchQueue.global().async {
defer { group.leave() }
do {
var service = try AwsService(
model.value,
endpoints: endpoints,
outputHTMLComments: self.command.htmlComments,
logger: self.logger
)
// get service filename without path and extension
let filename = model.key
.split(separator: "/", omittingEmptySubsequences: true).last!
let filenameWithoutExtension = String(filename[..<(filename.lastIndex(of: ".") ?? filename.endIndex)])

if let serviceConfig = config.services?[filenameWithoutExtension], let filter = serviceConfig.operations {
service.filterOperations(filter)
}
if self.command.output {
try self.generateFiles(with: service, config: config)

Smithy.registerShapesAndTraits()
Smithy.registerAWSTraits()
Smithy.registerTraitTypes(
SotoInputShapeTrait.self,
SotoOutputShapeTrait.self
)

try await withThrowingTaskGroup(of: Void.self) { group in
for file in modelFiles {
group.addTask {
do {
let model: SotoSmithy.Model
if self.command.smithy {
model = try self.loadSmithy(filename: file)
} else {
model = try self.loadJSONAST(filename: file)
}
var service = try AwsService(
model,
endpoints: endpoints,
outputHTMLComments: self.command.htmlComments,
logger: self.logger
)
// get service filename without path and extension
let filename = file
.split(separator: "/", omittingEmptySubsequences: true).last!
let filenameWithoutExtension = String(filename[..<(filename.lastIndex(of: ".") ?? filename.endIndex)])

if let serviceConfig = config.services?[filenameWithoutExtension], let filter = serviceConfig.operations {
service.filterOperations(filter)
}
if self.command.output {
try self.generateFiles(with: service, config: config)
}
} catch {
self.logger.error("\(file): \(error)")
exit(1)
}
} catch {
self.logger.error("\(model.key): \(error)")
exit(1)
}
}
try await group.waitForAll()
}

group.wait()

if models.count > 1 {
if modelFiles.count > 1 {
self.logger.info("Code Generation took \(Int(-startTime.timeIntervalSinceNow)) seconds")
}
}
Expand Down Expand Up @@ -151,33 +156,41 @@ public struct SotoCodeGen {
let modelFiles = self.getModelFiles()

return try .init(modelFiles.map {
do {
let data = try Data(contentsOf: URL(fileURLWithPath: $0))
let model = try Smithy().decodeAST(from: data)
try model.validate()
return (key: $0, value: model)
} catch {
throw FileError(filename: $0, error: error)
}
try (key: $0, value: self.loadJSONAST(filename: $0))
}) { left, _ in left }
}

func loadJSONAST(filename: String) throws -> SotoSmithy.Model {
do {
let data = try Data(contentsOf: URL(fileURLWithPath: filename))
let model = try Smithy().decodeAST(from: data)
try model.validate()
return model
} catch {
throw FileError(filename: filename, error: error)
}
}

func loadSmithy() throws -> [String: SotoSmithy.Model] {
let modelFiles = self.getSmithyFiles()

return try .init(modelFiles.map {
do {
let data = try Data(contentsOf: URL(fileURLWithPath: $0))
let string = String(decoding: data, as: Unicode.UTF8.self)
let model = try Smithy().parse(string)
try model.validate()
return (key: $0, value: model)
} catch {
throw FileError(filename: $0, error: error)
}
try (key: $0, value: self.loadSmithy(filename: $0))
}) { left, _ in left }
}

func loadSmithy(filename: String) throws -> SotoSmithy.Model {
do {
let data = try Data(contentsOf: URL(fileURLWithPath: filename))
let string = String(decoding: data, as: Unicode.UTF8.self)
let model = try Smithy().parse(string)
try model.validate()
return model
} catch {
throw FileError(filename: filename, error: error)
}
}

/// Generate service files from AWSService
/// - Parameter codeGenerator: service generated from JSON
func generateFiles(with service: AwsService, config: ConfigFile) throws {
Expand Down

0 comments on commit d27d783

Please sign in to comment.