Adding gyb discrepancy check to sanity script. (apple#13)

* 1) Adding gyb to scripts 2) Adding warning in gyb files about gyb overwritting manual edits to gyb generated swift files 3) Updated README with gyb instructions 4) Adding script for running all gyb generation 5) Adding CI check for discrepancies in gyb generated files * Updating Dockerfile
PADL · Feb 5, 2020 · de232cb · de232cb
1 parent 36ea42c
commit de232cb
Show file tree

Hide file tree

Showing 15 changed files with 1,366 additions and 20 deletions.
diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,4 @@ Package.resolved
 .podspecs
 DerivedData
 .swiftpm
+**/gyb.pyc
diff --git a/README.md b/README.md
@@ -42,7 +42,11 @@ Files that meet the above criteria are specific to the Swift Crypto implementati
 
 Files that do not have the `_boring` suffix are part of the public API of CryptoKit. Changing these requires passing a higher bar, as any change in these files must be accompanied by a change in CryptoKit itself.
 
-### Contributing New Primitives
+## Contributing
+
+Before contributing please read [CONTRIBUTING.md](CONTRIBUTING.md), also make sure to read the two following sections.
+
+#### Contributing new primitives
 
 To contribute a new cryptographic primitive to Swift Crypto, you should address the following questions:
 
@@ -54,14 +58,38 @@ To contribute a new cryptographic primitive to Swift Crypto, you should address
 
 In addition, new primitive implementations will only be accepted in cases where the implementation is thoroughly tested, including being tested with all currently available test vectors. If the [Wycheproof](https://github.com/google/wycheproof) project provides vectors for the algorithm those should be tested as well. It must be possible to ensure that we can appropriately regression test our implementations.
 
-### Contributing bug fixes
+#### Contributing bug fixes
 
 If you discover a bug with Swift Crypto, please report it via GitHub.
 
 If you are interested in fixing a bug, feel free to open a pull request. Please also submit regression tests with bug fixes to ensure that they are not regressed in future.
 
 If you have issues with CryptoKit, instead of Swift Crypto, please use [Feedback Assistant](https://feedbackassistant.apple.com) to file those issues as you normally would.
 
+### Get started contributing
+
+#### `gyb`
+
+Some of the files in this project are autogenerated (metaprogramming) using the Swift Utils tools called [gyb](https://github.com/apple/swift/blob/master/utils/gyb.py) (_"generate your boilerplate"_). `gyb` is included in [`./scripts/gyb`](scripts/gyb).
+
+`gyb` will generate some `Foobar.swift` Swift file from some `Foobar.swift.gyb` _template_ file. **You should not edit `Foobar.swift` directly**, since all manual edits in that generated file will be overwritten the next time `gyb` is run.
+
+You run `gyb` for a single file like so:
+
+```bash
+./scripts/gyb --line-directive "" Sources/Foobar.swift.gyb -o Sources/Foobar.swift
+```
+
+More conveniently you can run the bash script `./scripts/generate_boilerplate_files_with_gyb.sh` to generate all Swift files from their corresponding gyb template.
+
+**If you add a new `.gyb` file, you should append a `// MARK: - Generated file, do NOT edit` warning** inside it, e.g.
+
+```swift
+// MARK: - Generated file, do NOT edit
+// any edits of this file WILL be overwritten and thus discarded
+// see section `gyb` in `README` for details.
+```
+
 ### Security
 
 If you believe you have identified a vulnerability in Swift Crypto, please [report that vulnerability to Apple through the usual channel](https://support.apple.com/en-us/HT201220).

diff --git a/Sources/Crypto/AEADs/Nonces.swift b/Sources/Crypto/AEADs/Nonces.swift
@@ -15,6 +15,9 @@
 @_exported import CryptoKit
 #else
 import Foundation
+// MARK: - Generated file, do NOT edit
+// any edits of this file WILL be overwritten and thus discarded
+// see section `gyb` in `README` for details.
 
 extension AES.GCM {
     public struct Nonce: ContiguousBytes, Sequence {

diff --git a/Sources/Crypto/AEADs/Nonces.swift.gyb b/Sources/Crypto/AEADs/Nonces.swift.gyb
@@ -15,6 +15,9 @@
 @_exported import CryptoKit
 #else
 import Foundation
+// MARK: - Generated file, do NOT edit
+// any edits of this file WILL be overwritten and thus discarded
+// see section `gyb` in `README` for details.
 %{
 ciphers = [{"name": "AES.GCM", "recommendedNonceSize": "AES.GCM.defaultNonceByteCount", "nonceValidation": "< AES.GCM.defaultNonceByteCount"},{"name": "ChaChaPoly", "recommendedNonceSize": "ChaChaPoly.nonceByteCount", "nonceValidation": "!= ChaChaPoly.nonceByteCount"}]
 }%

diff --git a/Sources/Crypto/Digests/Digests.swift b/Sources/Crypto/Digests/Digests.swift
@@ -14,6 +14,9 @@
 #if (os(macOS) || os(iOS) || os(watchOS) || os(tvOS)) && CRYPTO_IN_SWIFTPM && !CRYPTO_IN_SWIFTPM_FORCE_BUILD_API
 @_exported import CryptoKit
 #else
+// MARK: - Generated file, do NOT edit
+// any edits of this file WILL be overwritten and thus discarded
+// see section `gyb` in `README` for details.
 
 @available(iOS 13.0, macOS 10.15, watchOS 6.0, tvOS 13.0, macCatalyst 13.0, *)
 public struct SHA256Digest: DigestPrivate {
@@ -61,6 +64,7 @@ public struct SHA256Digest: DigestPrivate {
     }
 }
 
+
 @available(iOS 13.0, macOS 10.15, watchOS 6.0, tvOS 13.0, macCatalyst 13.0, *)
 public struct SHA384Digest: DigestPrivate {
     let bytes: (UInt64, UInt64, UInt64, UInt64, UInt64, UInt64)
@@ -109,6 +113,7 @@ public struct SHA384Digest: DigestPrivate {
     }
 }
 
+
 @available(iOS 13.0, macOS 10.15, watchOS 6.0, tvOS 13.0, macCatalyst 13.0, *)
 public struct SHA512Digest: DigestPrivate {
     let bytes: (UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64)
@@ -205,7 +210,6 @@ public struct SHA1Digest: DigestPrivate {
     }
 }
 }
-
 extension Insecure {
 @available(iOS 13.0, macOS 10.15, watchOS 6.0, tvOS 13.0, macCatalyst 13.0, *)
 public struct MD5Digest: DigestPrivate {

diff --git a/Sources/Crypto/Digests/Digests.swift.gyb b/Sources/Crypto/Digests/Digests.swift.gyb
@@ -14,27 +14,26 @@
 #if (os(macOS) || os(iOS) || os(watchOS) || os(tvOS)) && CRYPTO_IN_SWIFTPM && !CRYPTO_IN_SWIFTPM_FORCE_BUILD_API
 @_exported import CryptoKit
 #else
-
+// MARK: - Generated file, do NOT edit
+// any edits of this file WILL be overwritten and thus discarded
+// see section `gyb` in `README` for details.
 %{
 digests_and_length = [{"name": "SHA256", "count": 32},{"name": "SHA384","count":48},{"name":"SHA512", "count": 64},{"name":"SHA1", "count":20, "prefix":"Insecure"},{"name":"MD5", "count":16, "prefix":"Insecure"}]
 }%
-
 % for HF in digests_and_length:
 %{
     name = HF["name"]
     byteCount = HF["count"]
     wordsCount = (byteCount*8)/64 + (0 if ((byteCount*8)%64) == 0 else 1)
 }%
-
 %{
     if "prefix" in HF.keys():
-        protocol_prefix = "extension " + HF["prefix"] + "{"
+        protocol_prefix = "extension " + HF["prefix"] + " {"
         protocol_suffix = "}"
     else:
         protocol_prefix = ""
         protocol_suffix = ""
 }%
-
 ${protocol_prefix}
 @available(iOS 13.0, macOS 10.15, watchOS 6.0, tvOS 13.0, macCatalyst 13.0, *)
 public struct ${name}Digest: DigestPrivate {
@@ -78,7 +77,7 @@ public struct ${name}Digest: DigestPrivate {
     }
 
     public func hash(into hasher: inout Hasher) {
-        self.withUnsafeBytes {hasher.combine(bytes: $0)}
+        self.withUnsafeBytes { hasher.combine(bytes: $0) }
     }
 }
 ${protocol_suffix}

diff --git a/Sources/Crypto/Key Agreement/ECDH.swift b/Sources/Crypto/Key Agreement/ECDH.swift
@@ -23,6 +23,9 @@ typealias NISTCurvePrivateKeyImpl = OpenSSLNISTCurvePrivateKeyImpl
 #endif
 
 import Foundation
+// MARK: - Generated file, do NOT edit
+// any edits of this file WILL be overwritten and thus discarded
+// see section `gyb` in `README` for details.
 
 extension P256 {
     public enum Signing {

diff --git a/Sources/Crypto/Key Agreement/ECDH.swift.gyb b/Sources/Crypto/Key Agreement/ECDH.swift.gyb
@@ -23,13 +23,13 @@ typealias NISTCurvePrivateKeyImpl = OpenSSLNISTCurvePrivateKeyImpl
 #endif
 
 import Foundation
-
-
+// MARK: - Generated file, do NOT edit
+// any edits of this file WILL be overwritten and thus discarded
+// see section `gyb` in `README` for details.
 %{
     NIST_CURVES = ["P256", "P384", "P521"]
     CURVES_FUNC = ["Signing", "KeyAgreement"]
 }%
-
 % for CURVE in NIST_CURVES:
 % for FUNC in CURVES_FUNC:
 
@@ -90,7 +90,6 @@ extension ${CURVE} {
 % end
 % end
 
-
 % for CURVE in NIST_CURVES:
 extension ${CURVE}.KeyAgreement.PrivateKey: DiffieHellmanKeyAgreement {
     /// Performs a key agreement with provided public key share.

diff --git a/Sources/Crypto/Signatures/ECDSA.swift b/Sources/Crypto/Signatures/ECDSA.swift
@@ -15,6 +15,9 @@
 @_exported import CryptoKit
 #else
 import Foundation
+// MARK: - Generated file, do NOT edit
+// any edits of this file WILL be overwritten and thus discarded
+// see section `gyb` in `README` for details.
 
 protocol NISTECDSASignature {
     init<D: DataProtocol>(rawRepresentation: D) throws

diff --git a/Sources/Crypto/Signatures/ECDSA.swift.gyb b/Sources/Crypto/Signatures/ECDSA.swift.gyb
@@ -15,7 +15,9 @@
 @_exported import CryptoKit
 #else
 import Foundation
-
+// MARK: - Generated file, do NOT edit
+// any edits of this file WILL be overwritten and thus discarded
+// see section `gyb` in `README` for details.
 %{
     NIST_CURVES_AND_HF = [{"curve": "P256", "hf": "SHA256"},{"curve": "P384", "hf": "SHA384"},{"curve": "P521", "hf": "SHA512"}]
 }%
@@ -38,7 +40,6 @@ protocol NISTSigning {
     CURVE = CURVE_AND_HF["curve"]
     HF    = CURVE_AND_HF["hf"]
 }%
-
 /// An ECDSA (Elliptic Curve Digital Signature Algorithm) Signature
 extension ${CURVE}.Signing {
     public struct ECDSASignature: ContiguousBytes, NISTECDSASignature {
@@ -67,8 +68,8 @@ extension ${CURVE}.Signing {
 
         var composite: (r: Data, s: Data) {
             let combined = rawRepresentation
-            assert(combined.count%2 == 0)
-            let half = combined.count/2
+            assert(combined.count % 2 == 0)
+            let half = combined.count / 2
             return (combined.prefix(upTo: half), combined.suffix(from: half))
         }
 
@@ -155,5 +156,4 @@ extension ${CURVE}.Signing.PublicKey: DigestValidator {
  }
 
 % end
-
 #endif // Linux or !SwiftPM
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -11,9 +11,12 @@ ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8
 
+# Get lldb out of the way
+RUN mv /usr/lib/python2.7/site-packages/* /usr/lib/python2.7/dist-packages && rmdir /usr/lib/python2.7/site-packages && ln -s /usr/lib/python2.7/dist-packages /usr/lib/python2.7/site-packages
+
 # dependencies
 RUN apt-get update && apt-get install -y wget
-RUN apt-get update && apt-get install -y lsof dnsutils netcat-openbsd net-tools curl jq # used by integration tests
+RUN apt-get update && apt-get install -y lsof dnsutils netcat-openbsd net-tools curl jq python2.7 # used by integration tests
 
 # ruby and jazzy for docs generation
 RUN apt-get update && apt-get install -y ruby ruby-dev libsqlite3-dev

diff --git a/scripts/generate_boilerplate_files_with_gyb.sh b/scripts/generate_boilerplate_files_with_gyb.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftCrypto open source project
+##
+## Copyright (c) 2019 Apple Inc. and the SwiftCrypto project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftCrypto project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+set -eu
+find . -name '*.gyb' |                                               \
+    while read file; do                                              \
+        ./scripts/gyb --line-directive '' -o "${file%.gyb}" "$file"; \
+    done
diff --git a/scripts/gyb b/scripts/gyb
@@ -0,0 +1,3 @@
+#!/usr/bin/env python2.7
+import gyb
+gyb.main()