Add include_file statement (#74)

README.md

@@ -174,7 +174,9 @@ assertEquals(onePlusOne, anotherOnePlusOne)
 // Top level
 type_universe ::= <stmt>...
 definition ::= '(' 'define' symbol <domain_definition> ')'
-stmt ::=  <definition> | <transform>
+stmt ::=  <definition> | <transform> | <include_file>
+
+include_file ::= `(include_file <path-to-file>)`
 
 // Domain
 domain_definition ::= <domain> | <permute_domain>
@@ -400,6 +402,43 @@ Unlike record elements, product element defintions must include identifiers.
 (product int_pair first::int second::int)
 ```
 
+#### Type Domain Includes
+
+It is possible to split type universes among multiple files, which allows type domains defined in another project to be
+permuted.  For example:
+
+```
+// root.ion:
+(include_file "sibling.ion")
+(include_file "sub-dir/thing.ion")
+(include_file "/other-project/toy-ast.ion")
+```
+
+While discussing further details, it is helpful to introduce two terms: an "includer" is a file which includes another
+using `include_file`, and the "includee" is a file which is included.
+
+The `root.ion` universe will contain all type domains from all includees and may still define additional type domains
+of its own.
+
+`include_file` statements are allowed in includees. Any attempt to include a file that has already been seen will be 
+ignored.
+
+When resolving the file to include, the following locations are searched:
+
+- The directory containing the includer (if the path to the includee does not start with `/`)
+- The directory containing the "main" type universe that was passed to `pig` on the command-line.
+- Any directories specified with the `--include` or `-I` arguments, in the order they were specified.
+
+The first matching file found wins and any other matching files ignored.
+
+Note that paths starting with `/` do not actually refer to the root of any file system, but instead are treated as 
+relative to the include directories.
+
+Paths specified with `include_file` may only contain alphanumeric or one of the following characters: 
+`-`, `_`, `.` or `/`.  Additionally, two consecutive periods ".." (i.e. a parent directory) are not allowed.
+
+Lastly, `include_file` can only be used at the top-level within a `.ion` file.  It is not allowed anywhere within a 
+`(domain ...)` clause.
 
 #### Using PIG In Your Project
 
@@ -408,16 +447,14 @@ Unlike record elements, product element defintions must include identifiers.
 At build time and before compilation of your application or library, the following should be executed:
 
 ```
-pig \
-    -u <type universe.ion> \
-    -t kotlin \ 
-    -n <namespace> \ 
-    -o path/to/package/<output file>
+pig -u <type universe.ion> -t kotlin -n <namespace> -o <path/to/output_file> [ -I <path-to-include-dir> ]  
 ```
 
 - `<type universe.ion>`:  path to the Ion text file containing the type universe
-- `<output file>`: path to the file for the generated code 
+- `<path/to/output_file>`: path to the file for the generated code 
 - `<namespace>`: the name used in the `package` statement at the top of the output file
+- `<path-to-include-dir>`: search path to external include directory (optional).  Can be specified more than once,
+i.e. `pig ... -I <dir1> -I <dir2> -I <dir3>`
 
 Execute: `pig --help` for all command-line options.
 

pig/build.gradle

@@ -43,6 +43,7 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.6.2'
     testImplementation 'org.junit.jupiter:junit-jupiter-params:5.6.2'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.6.2'
+    testImplementation 'com.google.jimfs:jimfs:1.2'
 }
 
 application {

pig/src/org/partiql/pig/cmdline/Command.kt

@@ -15,10 +15,36 @@
 
 package org.partiql.pig.cmdline
 
-import java.io.File
+import java.nio.file.Path
 
+/** Represents command line options specified by the user. */
 sealed class Command {
+
+    /** The `--help` command. */
     object ShowHelp : Command()
+
+    /**
+     * Returned by [CommandLineParser] when the user has specified invalid command-line arguments
+     *
+     * - [message]: an error message to be displayed to the user.
+     */
     data class InvalidCommandLineArguments(val message: String) : Command()
-    data class Generate(val typeUniverseFile: File, val outputFile: File, val target: TargetLanguage) : Command()
-}
+
+    /**
+     * Contains the details of a *valid* command-line specified by the user.
+     *
+     * - [typeUniverseFilePath]: the path to the type universe file.
+     * - [outputFilePath]: the path to the output file.  (This makes the assumption that there is only one output file.)
+     * - [includePaths]: directories to be searched when looking for files included with `include_file`.
+     * - [target]: specifies the target language and any other parameters unique to the target language.
+     */
+    data class Generate(
+        val typeUniverseFilePath: Path,
+        val outputFilePath: Path,
+        val includePaths: List<Path>,
+        val target: TargetLanguage
+    ) : Command()
+}
+
+
+

pig/src/org/partiql/pig/cmdline/CommandLineParser.kt

@@ -16,8 +16,9 @@
 package org.partiql.pig.cmdline
 
 import joptsimple.*
-import java.io.File
 import java.io.PrintStream
+import java.nio.file.Path
+import java.nio.file.Paths
 
 
 class CommandLineParser {
@@ -30,7 +31,7 @@ class CommandLineParser {
         HTML
     }
 
-    private object languageTargetTypeValueConverter : ValueConverter<LanguageTargetType> {
+    private object LanguageTargetTypeValueConverter : ValueConverter<LanguageTargetType> {
         private val lookup = LanguageTargetType.values().associateBy { it.name.toLowerCase() }
 
         override fun convert(value: String?): LanguageTargetType {
@@ -47,6 +48,12 @@ class CommandLineParser {
         }
     }
 
+    private object PathValueConverter : ValueConverter<Path> {
+        override fun convert(value: String?): Path = Paths.get(value).toAbsolutePath().normalize()
+        override fun valueType(): Class<out Path> = Path::class.java
+        override fun valuePattern(): String? = null
+    }
+
     private val formatter = object : BuiltinHelpFormatter(120, 2) {
         override fun format(options: MutableMap<String, out OptionDescriptor>?): String {
             return """PartiQL I.R. Generator
@@ -56,6 +63,8 @@ class CommandLineParser {
                 |
                 |  --target=kotlin requires --namespace=<ns>
                 |  --target=custom requires --template=<path-to-template>
+                |  All paths specified in these command-line options are relative to the current working
+                |  directory by default.
                 | 
                 |Examples:
                 |
@@ -65,26 +74,32 @@ class CommandLineParser {
         """.trimMargin()
         }
     }
-    private val optParser = OptionParser().also { it.formatHelpWith(formatter) }
+    private val optParser = OptionParser().apply {
+        formatHelpWith(formatter)
+    }
 
 
     private val helpOpt = optParser.acceptsAll(listOf("help", "h", "?"), "prints this help")
         .forHelp()
 
     private val universeOpt = optParser.acceptsAll(listOf("universe", "u"), "Type universe input file")
         .withRequiredArg()
-        .ofType(File::class.java)
+        .withValuesConvertedBy(PathValueConverter)
         .required()
 
+    private val includeSearchRootOpt = optParser.acceptsAll(listOf("include", "I"), "Include search path")
+        .withRequiredArg()
+        .withValuesConvertedBy(PathValueConverter)
+        .describedAs("Search path for files included with include_file. May be specified multiple times.")
+
     private val outputOpt = optParser.acceptsAll(listOf("output", "o"), "Generated output file")
         .withRequiredArg()
-        .ofType(File::class.java)
+        .withValuesConvertedBy(PathValueConverter)
         .required()
 
     private val targetTypeOpt = optParser.acceptsAll(listOf("target", "t"), "Target language")
         .withRequiredArg()
-        //.ofType(LanguageTargetType::class.java)
-        .withValuesConvertedBy(languageTargetTypeValueConverter)
+        .withValuesConvertedBy(LanguageTargetTypeValueConverter)
         .required()
 
     private val namespaceOpt = optParser.acceptsAll(listOf("namespace", "n"), "Namespace for generated code")
@@ -93,7 +108,7 @@ class CommandLineParser {
 
     private val templateOpt = optParser.acceptsAll(listOf("template", "e"), "Path to an Apache FreeMarker template")
         .withOptionalArg()
-        .ofType(File::class.java)
+        .withValuesConvertedBy(PathValueConverter)
 
 
     /**
@@ -116,9 +131,15 @@ class CommandLineParser {
                 optSet.has(helpOpt) -> Command.ShowHelp
                 else -> {
                     // !! is fine in this case since we define these options as .required() above.
-                    val typeUniverseFile: File = optSet.valueOf(universeOpt)!!
+                    val typeUniverseFile: Path = optSet.valueOf(universeOpt)!!
                     val targetType = optSet.valueOf(targetTypeOpt)!!
-                    val outputFile: File = optSet.valueOf(outputOpt)!!
+                    val outputFile: Path = optSet.valueOf(outputOpt)!!
+
+                    // Always add the parent of the file containing the main type universe as an include root.
+                    val includeSearchRoots = listOf(
+                        typeUniverseFile.parent,
+                        *optSet.valuesOf(includeSearchRootOpt)!!.toTypedArray()
+                    )
 
                     if (targetType.requireNamespace) {
                         if (!optSet.has(namespaceOpt)) {
@@ -145,13 +166,11 @@ class CommandLineParser {
                         LanguageTargetType.CUSTOM -> TargetLanguage.Custom(optSet.valueOf(templateOpt))
                     }
 
-                    Command.Generate(typeUniverseFile, outputFile, target)
+                    Command.Generate(typeUniverseFile, outputFile, includeSearchRoots, target)
                 }
             }
         } catch(ex: OptionException) {
             Command.InvalidCommandLineArguments(ex.message!!)
         }
-
     }
-
-}
+}

pig/src/org/partiql/pig/cmdline/TargetLanguage.kt

@@ -16,9 +16,10 @@
 package org.partiql.pig.cmdline
 
 import java.io.File
+import java.nio.file.Path
 
 sealed class TargetLanguage {
     data class Kotlin(val namespace: String) : TargetLanguage()
-    data class Custom(val templateFile: File) : TargetLanguage()
+    data class Custom(val templateFile: Path) : TargetLanguage()
     object Html : TargetLanguage()
 }

pig/src/org/partiql/pig/domain/model/SemanticErrorContext.kt

@@ -15,12 +15,12 @@
 
 package org.partiql.pig.domain.model
 
-import com.amazon.ionelement.api.IonLocation
 import com.amazon.ionelement.api.MetaContainer
-import com.amazon.ionelement.api.location
-import org.partiql.pig.errors.PigException
+import org.partiql.pig.domain.parser.SourceLocation
+import org.partiql.pig.domain.parser.sourceLocation
 import org.partiql.pig.errors.ErrorContext
 import org.partiql.pig.errors.PigError
+import org.partiql.pig.errors.PigException
 
 /**
  * Encapsulates all error context information in an easily testable way.
@@ -66,16 +66,16 @@ sealed class SemanticErrorContext(val msgFormatter: () -> String): ErrorContext
         : SemanticErrorContext({ "Cannot remove built-in type '$typeName'" })
 
     data class DuplicateTypeDomainName(val domainName: String)
-        : SemanticErrorContext({ "Duplicate type domain tag: '${domainName} "})
+        : SemanticErrorContext({ "Duplicate type domain tag: '${domainName}' "})
 
     data class DuplicateRecordElementTag(val elementName: String)
-        : SemanticErrorContext({ "Duplicate record element tag: '${elementName} "})
+        : SemanticErrorContext({ "Duplicate record element tag: '${elementName}' "})
 
     data class DuplicateElementIdentifier(val elementName: String)
-        : SemanticErrorContext({ "Duplicate element identifier: '${elementName} "})
+        : SemanticErrorContext({ "Duplicate element identifier: '${elementName}' "})
 
     data class NameAlreadyUsed(val name: String, val domainName: String)
-        : SemanticErrorContext({ "Name '$name' was previously used in the `$domainName` type domain" })
+        : SemanticErrorContext({ "Name '$name' was previously used in the '$domainName' type domain" })
 
     data class CannotRemoveNonExistentSumVariant(val sumTypeName: String, val variantName: String)
         : SemanticErrorContext({ "Permuted sum type '${sumTypeName}' tries to remove variant '${variantName}' which " +
@@ -109,9 +109,9 @@ sealed class SemanticErrorContext(val msgFormatter: () -> String): ErrorContext
  * Shortcut for throwing [PigException] with the specified metas and [PigError].
  */
 fun semanticError(blame: MetaContainer, context: ErrorContext): Nothing =
-    semanticError(blame.location, context)
+    semanticError(blame.sourceLocation, context)
 /**
  * Shortcut for throwing [PigException] with the specified metas and [PigError].
  */
-fun semanticError(blame: IonLocation?, context: ErrorContext): Nothing =
+fun semanticError(blame: SourceLocation?, context: ErrorContext): Nothing =
     throw PigException(PigError(blame, context))

pig/src/org/partiql/pig/domain/parser/ParserErrorContext.kt

@@ -15,14 +15,12 @@
 
 package org.partiql.pig.domain.parser
 
-import com.amazon.ionelement.api.IonElement
-import com.amazon.ionelement.api.IonLocation
-import com.amazon.ionelement.api.location
 import com.amazon.ionelement.api.ElementType
 import com.amazon.ionelement.api.IonElementException
-import org.partiql.pig.errors.PigException
 import org.partiql.pig.errors.ErrorContext
 import org.partiql.pig.errors.PigError
+import org.partiql.pig.errors.PigException
+import javax.swing.text.html.parser.Parser
 
 /**
  * Variants of [ParserErrorContext] contain details about various parse errors that can be encountered
@@ -33,7 +31,7 @@ import org.partiql.pig.errors.PigError
 sealed class ParserErrorContext(val msgFormatter: () -> String): ErrorContext {
     override val message: String get() = msgFormatter()
 
-    /** Indicates that an []IonElectrolyteException] was thrown during parsing of a type universe. */
+    /** Indicates that an [IonElementException] was thrown during parsing of a type universe. */
     data class IonElementError(val ex: IonElementException)
         : ParserErrorContext({ ex.message!! }) {
         // This is for unit tests... we don't include IonElectrolyteException here since it doesn't implement
@@ -51,15 +49,29 @@ sealed class ParserErrorContext(val msgFormatter: () -> String): ErrorContext {
     data class InvalidTopLevelTag(val tag: String)
         : ParserErrorContext({ "Invalid top-level tag: '$tag'"})
 
-    data class InvalidSumLevelTag(val tag: String)
-        : ParserErrorContext({ "Invalid tag for sum variant: '$tag'"})
-
     data class InvalidPermutedDomainTag(val tag: String)
         : ParserErrorContext({ "Invalid tag for permute_domain body: '$tag'"})
 
     data class InvalidWithSumTag(val tag: String)
         : ParserErrorContext({ "Invalid tag for with body: '$tag'"})
 
+    data class IncludeFileNotFound(val includeFilePath: String, val searchedPaths: List<String> )
+        : ParserErrorContext(
+        {
+            "Could not locate include file '$includeFilePath' at any of the following locations:\n" +
+            searchedPaths.joinToString("\n")
+        }
+    )
+
+    data class IncludeFilePathContainsIllegalCharacter(val c: Char)
+        : ParserErrorContext({ "Illegal character '$c' in include_file path" })
+
+    object IncludeFilePathContainsParentDirectory
+        : ParserErrorContext({ "include_file path contained parent directory, i.e. \"..\"" })
+
+    object IncludeFilePathMustNotStartWithRoot
+        : ParserErrorContext({ "include_file path must not start with '/'" })
+
     data class ExpectedTypeReferenceArityTag(val tag: String)
         : ParserErrorContext({ "Expected '*' or '?' but found '$tag'"})
 
@@ -79,8 +91,7 @@ sealed class ParserErrorContext(val msgFormatter: () -> String): ErrorContext {
         : ParserErrorContext({ "Element has multiple name annotations"})
 }
 
-
-fun parseError(blame: IonLocation?, context: ErrorContext): Nothing =
+fun parseError(blame: SourceLocation?, context: ErrorContext): Nothing =
     PigError(blame, context).let {
         throw when (context) {
             is ParserErrorContext.IonElementError -> {
@@ -91,8 +102,3 @@ fun parseError(blame: IonLocation?, context: ErrorContext): Nothing =
         }
     }
 
-fun parseError(blame: IonElement, context: ErrorContext): Nothing {
-    val loc = blame.metas.location
-    parseError(loc, context)
-}
-