Add support for emoji and underscore in Swift identifiers

kayak · Nov 6, 2018 · 2cc45ce · 2cc45ce
1 parent 2ecd0c1
commit 2cc45ce
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 17 deletions.
diff --git a/SwiftGenStrings/SwiftLanguageToken.swift b/SwiftGenStrings/SwiftLanguageToken.swift
@@ -1,9 +1,9 @@
 import Foundation
 
 enum SwiftLanguageToken: Equatable {
-    case identifier(identifier: String) // import, var, ClassName, functionName
+    case identifier(_ identifier: String) // import, var, ClassName, functionName
     case text(text: String) // "...."
-    case comment(comment: String) // /* or //
+    case comment(_ comment: String) // /* or //
     case braceOpen // {
     case braceClose // }
     case bracketOpen // [

diff --git a/SwiftGenStrings/SwiftTokenizer.swift b/SwiftGenStrings/SwiftTokenizer.swift
@@ -78,29 +78,36 @@ class SwiftTokenizer {
                 while let next = iterator.next, next != "\n" {
                     comment = comment + String(next)
                 }
-                tokens.append(.comment(comment: comment))
+                tokens.append(.comment(comment))
 
             case _ where isIdentifierCharacter(character):
                 var identifier = String(character)
                 while let peek = iterator.peekNext, isIdentifierCharacter(peek) {
                     identifier = identifier + String(peek)
                     _ = iterator.next
                 }
-                tokens.append(.identifier(identifier: identifier))
+                tokens.append(.identifier(identifier))
 
             default:
-                tokens.append(.identifier(identifier: String(character)))
+                tokens.append(.identifier(String(character)))
             }
         }
         return tokens
     }
 
-    private let alphaNumericCharacterSet = CharacterSet.alphanumerics
-    private let optionalCharacterSet = CharacterSet(charactersIn: "?!")
+    private let allowedIdentifierCharacterSet: CharacterSet = {
+        let optionalCharacterSet = CharacterSet(charactersIn: "?!")
+        let underscores = CharacterSet(charactersIn: "_")
+
+        return CharacterSet.alphanumerics
+            .union(.symbols)
+            .union(optionalCharacterSet)
+            .union(underscores)
+    }()
 
     private func isIdentifierCharacter(_ character: Character) -> Bool {
-        for unichar in String(character).utf16 {
-            if !alphaNumericCharacterSet.contains(UnicodeScalar(unichar)!) && !optionalCharacterSet.contains(UnicodeScalar(unichar)!) {
+        for unicodeScalar in character.unicodeScalars {
+            if !allowedIdentifierCharacterSet.contains(unicodeScalar) {
                 return false
             }
         }

diff --git a/SwiftGenStringsTests/LocalizedStringFinderTests.swift b/SwiftGenStringsTests/LocalizedStringFinderTests.swift
@@ -5,27 +5,27 @@ class LocalizedStringFinderTests: XCTestCase {
     func testFindStringsWithTableNameAndBundle() {
         let finder = LocalizedStringFinder()
         let tokens: [SwiftLanguageToken] = [
-            .identifier(identifier: "NSLocalizedString"),
+            .identifier("NSLocalizedString"),
             .parenthesisOpen,
             .text(text: "KEY"),
             .comma,
-            .identifier(identifier: "tableName"),
+            .identifier("tableName"),
             .colon,
-            .identifier(identifier: "nil"),
+            .identifier("nil"),
             .comma,
-            .identifier(identifier: "bundle"),
+            .identifier("bundle"),
             .colon,
-            .identifier(identifier: "NSBundle"),
+            .identifier("NSBundle"),
             .dot,
-            .identifier(identifier: "mainBundle"),
+            .identifier("mainBundle"),
             .parenthesisOpen,
             .parenthesisClose,
             .comma,
-            .identifier(identifier: "value"),
+            .identifier("value"),
             .colon,
             .text(text: "VALUE"),
             .comma,
-            .identifier(identifier: "comment"),
+            .identifier("comment"),
             .colon,
             .text(text: "COMMENT"),
             .parenthesisClose

diff --git a/SwiftGenStringsTests/RealWorldStringsTests.swift b/SwiftGenStringsTests/RealWorldStringsTests.swift
@@ -40,6 +40,13 @@ class RealWorldStringsTests: XCTestCase {
                 comments: ["c"]),
             in: "NSLocalizedString(\"\"\"\n\tHere is some multi-line text \\\n\tMore text here\n\t\"\"\", comment: \"c\")")
     }
+
+    func testEmojiInIdentifierWithoutLocalizedString() {
+        let string = "var fontWeight📙: String"
+        verifyNoLocalizedString(in: string)
+        XCTAssertEqual(0, errorOutput.invalidIdentifiers.count)
+        XCTAssertEqual(0, errorOutput.invalidUnicodeCodePoints.count)
+    }
 
     // MARK: - Helpers
 

diff --git a/SwiftGenStringsTests/SwiftTokenizerTests.swift b/SwiftGenStringsTests/SwiftTokenizerTests.swift
@@ -34,6 +34,18 @@ class SwiftTokenizerTests: XCTestCase {
         XCTAssertEqual(6, tokens.count)
     }
 
+    func testTokenizerWithUnderscore() {
+        let string = "var _secret = \"123\""
+        let tokens = tokenizer.tokenizeSwiftString(string)
+        assertEqualTokens([.identifier("var"), .identifier("_secret"), .identifier("="), .text(text: "123")], tokens)
+    }
+
+    func testTokenizerWithEmojiInIdentifier() {
+        let string = "var fontWeight📙: String"
+        let tokens = tokenizer.tokenizeSwiftString(string)
+        assertEqualTokens([.identifier("var"), .identifier("fontWeight📙"), .colon, .identifier("String")], tokens)
+    }
+
     func testTokenizerWithComment() {
         let string = "// var foo = bar"
         let tokens = tokenizer.tokenizeSwiftString(string)
@@ -59,5 +71,15 @@ class SwiftTokenizerTests: XCTestCase {
         XCTAssertEqual(8, tokens.count)
         XCTAssertEqual(tokens[2], SwiftLanguageToken.text(text: "Here is some multi-line text\\nwith newline"))
     }
+
+    // MARK: - Helpers
+
+    private func assertEqualTokens(_ expected: [SwiftLanguageToken], _ actual: [SwiftLanguageToken], file: StaticString = #file, line: UInt = #line) {
+        XCTAssertEqual(expected.count, actual.count, file: file, line: line)
+
+        for value in zip(expected, actual).enumerated() {
+            XCTAssertEqual(value.element.0, value.element.1, "Expected same tokens at index \(value.offset)", file: file, line: line)
+        }
+    }
 
 }