From 256bd8bc463ff36322b355b3819b7b23ec1f48d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20St=C3=BChrk?= Date: Sat, 1 May 2021 17:34:30 +0200 Subject: [PATCH] Add abstractions for end-to-end tests. --- Package.swift | 1 + .../GenerateTestCase.swift | 172 ++++++++++++++++ .../GeneratedDocumentation.swift | 184 ++++++++++++++++++ Tests/EndToEndTests/VisibilityTests.swift | 39 ++++ 4 files changed, 396 insertions(+) create mode 100644 Tests/EndToEndTests/Supporting Types for Generate Tests/GenerateTestCase.swift create mode 100644 Tests/EndToEndTests/Supporting Types for Generate Tests/GeneratedDocumentation.swift create mode 100644 Tests/EndToEndTests/VisibilityTests.swift diff --git a/Package.swift b/Package.swift index 69da9fa9..2517354a 100644 --- a/Package.swift +++ b/Package.swift @@ -71,6 +71,7 @@ let package = Package( name: "EndToEndTests", dependencies: [ .target(name: "swift-doc"), + .product(name: "Markup", package: "Markup"), ] ), ] diff --git a/Tests/EndToEndTests/Supporting Types for Generate Tests/GenerateTestCase.swift b/Tests/EndToEndTests/Supporting Types for Generate Tests/GenerateTestCase.swift new file mode 100644 index 00000000..fef4c668 --- /dev/null +++ b/Tests/EndToEndTests/Supporting Types for Generate Tests/GenerateTestCase.swift @@ -0,0 +1,172 @@ +import XCTest +import Foundation + +/// A class that provides abstractions to write tests for the `generate` subcommand. +/// +/// Create a subclass of this class to write test cases for the `generate` subcommand. +/// It provides an API to create source files which should be included in the sources. +/// Then you can generate the documentation. +/// If there's an error while generating the documentation for any of the formats, +/// the test automatically fails. +/// Additionally, it provides APIs to assert validations on the generated documentation. +/// +/// ``` swift +/// class TestVisibility: GenerateTestCase { +/// func testClassesVisibility() { +/// sourceFile("Example.swift") { +/// #""" +/// public class PublicClass {} +/// +/// class InternalClass {} +/// +/// private class PrivateClass {} +/// """# +/// } +/// +/// generate(minimumAccessLevel: .internal) +/// +/// XCTAssertDocumentationContains(.class("PublicClass")) +/// XCTAssertDocumentationContains(.class("InternalClass")) +/// XCTAssertDocumentationNotContains(.class("PrivateClass")) +/// } +/// } +/// ``` +/// +/// The tests are end-to-end tests. +/// They use the command-line tool to build the documentation +/// and run the assertions +/// by reading and understanding the created output of the documentation. +class GenerateTestCase: XCTestCase { + private var sourcesDirectory: URL? + + private var outputs: [GeneratedDocumentation] = [] + + /// The output formats which should be generated for this test case. + /// You can set a new value in `setUp()` if a test should only generate specific formats. + var testedOutputFormats: [GeneratedDocumentation.Type] = [] + + override func setUpWithError() throws { + try super.setUpWithError() + + sourcesDirectory = try createTemporaryDirectory() + + testedOutputFormats = [GeneratedHTMLDocumentation.self, GeneratedCommonMarkDocumentation.self] + } + + override func tearDown() { + super.tearDown() + + if let sourcesDirectory = self.sourcesDirectory { + try? FileManager.default.removeItem(at: sourcesDirectory) + } + for output in outputs { + try? FileManager.default.removeItem(at: output.directory) + } + } + + func sourceFile(_ fileName: String, contents: () -> String, file: StaticString = #filePath, line: UInt = #line) { + guard let sourcesDirectory = self.sourcesDirectory else { + return assertionFailure() + } + do { + try contents().write(to: sourcesDirectory.appendingPathComponent(fileName), atomically: true, encoding: .utf8) + } + catch let error { + XCTFail("Could not create source file '\(fileName)' (\(error))", file: file, line: line) + } + } + + func generate(minimumAccessLevel: MinimumAccessLevel, file: StaticString = #filePath, line: UInt = #line) { + for format in testedOutputFormats { + do { + let outputDirectory = try createTemporaryDirectory() + try Process.run(command: swiftDocCommand, + arguments: [ + "generate", + "--module-name", "SwiftDoc", + "--format", format.outputFormat, + "--output", outputDirectory.path, + "--minimum-access-level", minimumAccessLevel.rawValue, + sourcesDirectory!.path + ]) { result in + if result.terminationStatus != EXIT_SUCCESS { + XCTFail("Generating documentation failed for format \(format.outputFormat)", file: file, line: line) + } + } + + outputs.append(format.init(directory: outputDirectory)) + } + catch let error { + XCTFail("Could not generate documentation format \(format.outputFormat) (\(error))", file: file, line: line) + } + } + } +} + + +extension GenerateTestCase { + func XCTAssertDocumentationContains(_ symbolType: SymbolType, file: StaticString = #filePath, line: UInt = #line) { + for output in outputs { + if output.symbol(symbolType) == nil { + XCTFail("Output \(type(of: output).outputFormat) is missing \(symbolType)", file: file, line: line) + } + } + } + + func XCTAssertDocumentationNotContains(_ symbolType: SymbolType, file: StaticString = #filePath, line: UInt = #line) { + for output in outputs { + if output.symbol(symbolType) != nil { + XCTFail("Output \(type(of: output).outputFormat) contains \(symbolType) although it should be omitted", file: file, line: line) + } + } + } + + enum SymbolType: CustomStringConvertible { + case `class`(String) + case `struct`(String) + case `enum`(String) + case `typealias`(String) + case `protocol`(String) + case function(String) + case variable(String) + case `extension`(String) + + var description: String { + switch self { + case .class(let name): + return "class '\(name)'" + case .struct(let name): + return "struct '\(name)'" + case .enum(let name): + return "enum '\(name)'" + case .typealias(let name): + return "typealias '\(name)'" + case .protocol(let name): + return "protocol '\(name)'" + case .function(let name): + return "func '\(name)'" + case .variable(let name): + return "variable '\(name)'" + case .extension(let name): + return "extension '\(name)'" + } + } + } +} + + +extension GenerateTestCase { + + enum MinimumAccessLevel: String { + case `public`, `internal`, `private` + } +} + + + +private func createTemporaryDirectory() throws -> URL { + let temporaryDirectoryURL = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: temporaryDirectoryURL, withIntermediateDirectories: true) + + return temporaryDirectoryURL +} diff --git a/Tests/EndToEndTests/Supporting Types for Generate Tests/GeneratedDocumentation.swift b/Tests/EndToEndTests/Supporting Types for Generate Tests/GeneratedDocumentation.swift new file mode 100644 index 00000000..0f21b927 --- /dev/null +++ b/Tests/EndToEndTests/Supporting Types for Generate Tests/GeneratedDocumentation.swift @@ -0,0 +1,184 @@ +import Foundation +import HTML +import CommonMark + +/// A protocol which needs to be implemented by the different documentation generators. It provides an API to operate +/// on the generated documentation. +protocol GeneratedDocumentation { + + /// The name of the output format. This needs to be the name name like the value passed to swift-doc's `format` option. + static var outputFormat: String { get } + + init(directory: URL) + + var directory: URL { get } + + func symbol(_ symbolType: GenerateTestCase.SymbolType) -> Page? +} + +protocol Page { + var type: String? { get } + + var name: String? { get } +} + + + +struct GeneratedHTMLDocumentation: GeneratedDocumentation { + + static let outputFormat = "html" + + let directory: URL + + func symbol(_ symbolType: GenerateTestCase.SymbolType) -> Page? { + switch symbolType { + case .class(let name): + return page(for: name, ofType: "Class") + case .typealias(let name): + return page(for: name, ofType: "Typealias") + case .struct(let name): + return page(for: name, ofType: "Structure") + case .enum(let name): + return page(for: name, ofType: "Enumeration") + case .protocol(let name): + return page(for: name, ofType: "Protocol") + case .function(let name): + return page(for: name, ofType: "Function") + case .variable(let name): + return page(for: name, ofType: "Variable") + case .extension(let name): + return page(for: name, ofType: "Extensions on") + } + } + + private func page(for symbolName: String, ofType type: String) -> Page? { + guard let page = page(named: symbolName) else { return nil } + guard page.type == type else { return nil } + + return page + } + + private func page(named name: String) -> HtmlPage? { + let fileUrl = directory.appendingPathComponent(fileName(forSymbol: name)).appendingPathComponent("index.html") + guard + FileManager.default.isReadableFile(atPath: fileUrl.path), + let contents = try? String(contentsOf: fileUrl), + let document = try? HTML.Document(string: contents) + else { return nil } + + return HtmlPage(document: document) + } + + private func fileName(forSymbol symbolName: String) -> String { + symbolName + .replacingOccurrences(of: ".", with: "_") + .replacingOccurrences(of: " ", with: "-") + .components(separatedBy: reservedCharactersInFilenames).joined(separator: "_") + } + + private struct HtmlPage: Page { + let document: HTML.Document + + var type: String? { + let results = document.search(xpath: "//h1/small") + assert(results.count == 1) + return results.first?.content + } + + var name: String? { + let results = document.search(xpath: "//h1/code") + assert(results.count == 1) + return results.first?.content + } + } +} + + +struct GeneratedCommonMarkDocumentation: GeneratedDocumentation { + + static let outputFormat = "commonmark" + + let directory: URL + + func symbol(_ symbolType: GenerateTestCase.SymbolType) -> Page? { + switch symbolType { + case .class(let name): + return page(for: name, ofType: "class") + case .typealias(let name): + return page(for: name, ofType: "typealias") + case .struct(let name): + return page(for: name, ofType: "struct") + case .enum(let name): + return page(for: name, ofType: "enum") + case .protocol(let name): + return page(for: name, ofType: "protocol") + case .function(let name): + return page(for: name, ofType: "func") + case .variable(let name): + return page(for: name, ofType: "var") ?? page(for: name, ofType: "let") + case .extension(let name): + return page(for: name, ofType: "extension") + } + } + + private func page(for symbolName: String, ofType type: String) -> Page? { + guard let page = page(named: symbolName) else { return nil } + guard page.type == type else { return nil } + + return page + } + + private func page(named name: String) -> CommonMarkPage? { + let fileUrl = directory.appendingPathComponent("\(name).md") + guard + FileManager.default.isReadableFile(atPath: fileUrl.path), + let contents = try? String(contentsOf: fileUrl), + let document = try? CommonMark.Document(contents) + else { return nil } + + return CommonMarkPage(document: document) + } + + private func fileName(forSymbol symbolName: String) -> String { + symbolName + .replacingOccurrences(of: ".", with: "_") + .replacingOccurrences(of: " ", with: "-") + .components(separatedBy: reservedCharactersInFilenames).joined(separator: "_") + } + + private struct CommonMarkPage: Page { + let document: CommonMark.Document + + private var headingElement: Heading? { + document.children.first(where: { ($0 as? Heading)?.level == 1 }) as? Heading + } + + var type: String? { + // Our CommonMark pages don't give a hint of the actual type of a documentation page. That's why we extract + // it via a regex out of the declaration. Not very nice, but works for now. + guard + let name = self.name, + let code = document.children.first(where: { $0 is CodeBlock}) as? CodeBlock, + let codeContents = code.literal, + let extractionRegex = try? NSRegularExpression(pattern: "([a-z]+) \(name)") + else { return nil } + + guard + let match = extractionRegex.firstMatch(in: codeContents, range: NSRange(location: 0, length: codeContents.utf16.count)), + match.numberOfRanges > 0, + let range = Range(match.range(at: 1), in: codeContents) + else { return nil } + + return String(codeContents[range]) + } + + var name: String? { + headingElement?.children.compactMap { ($0 as? Literal)?.literal }.joined() + } + } +} + +private let reservedCharactersInFilenames: CharacterSet = [ + // Windows Reserved Characters + "<", ">", ":", "\"", "/", "\\", "|", "?", "*", +] diff --git a/Tests/EndToEndTests/VisibilityTests.swift b/Tests/EndToEndTests/VisibilityTests.swift new file mode 100644 index 00000000..c3f854d2 --- /dev/null +++ b/Tests/EndToEndTests/VisibilityTests.swift @@ -0,0 +1,39 @@ +import XCTest + +class TestVisibility: GenerateTestCase { + func testClassesVisibility() { + sourceFile("Example.swift") { + #""" + public class PublicClass {} + + class InternalClass {} + + private class PrivateClass {} + """# + } + + generate(minimumAccessLevel: .internal) + + XCTAssertDocumentationContains(.class("PublicClass")) + XCTAssertDocumentationContains(.class("InternalClass")) + XCTAssertDocumentationNotContains(.class("PrivateClass")) + } + + /// This example fails (because the tests are wrong, not because of a bug in `swift-doc`). + func testFailingExample() { + sourceFile("Example.swift") { + #""" + public class PublicClass {} + + public class AnotherPublicClass {} + + class InternalClass {} + """# + } + + generate(minimumAccessLevel: .public) + + XCTAssertDocumentationContains(.class("PublicClass")) + XCTAssertDocumentationContains(.class("InternalClass")) + } +}