Initial version

.bazelrc

@@ -0,0 +1 @@
+build --build_python_zip

.circleci/config.yml

@@ -0,0 +1,186 @@
+version: 2.1
+
+executors:
+  vs2019:
+    description: >
+      An executor preloaded with visual studios 2019 plus a number of other
+      development tools.
+    parameters:
+      version:
+        type: string
+        description: The image version to use when executing. Defaults to "201908-06"
+        default: "201908-06"
+      shell:
+        type: string
+        description: >
+          The shell to use.
+          Defaults to `powershell.exe -ExecutionPolicy Bypass`
+        default: powershell.exe -ExecutionPolicy Bypass
+    machine:
+      image: "windows-server-2019-vs2019:<< parameters.version >>"
+      resource_class: windows.medium
+      shell: << parameters.shell >>
+
+jobs:
+  test-linux:
+    docker:
+      - image: hchauvin/dsk-build:latest
+    steps:
+      - checkout
+      - run: bazel build //...
+      - run: bazel test //...
+      - run: mv bazel-bin/monorepo_tools.zip monorepo_tools_linux_amd64.zip
+      - persist_to_workspace:
+          root: '.'
+          paths:
+            - monorepo_tools_linux_amd64.zip
+  test-linux-standalone:
+    docker:
+      - image: hchauvin/dsk-build:latest
+    steps:
+      - attach_workspace:
+          at: ./artifacts
+      - run: python ./artifacts/monorepo_tools_linux_amd64.zip --help
+      - run: python2 ./artifacts/monorepo_tools_linux_amd64.zip --help
+  test-windows-py3:
+    executor:
+      name: vs2019
+    steps:
+      - checkout
+      - run: choco install --no-progress bazel
+      - run: choco install --no-progress python
+      - run: pip install wheel
+      - run: bazel build //...
+      - run: bazel test //...
+      - run: mv bazel-bin/monorepo_tools.zip monorepo_tools_py3_windows_amd64.zip
+      - persist_to_workspace:
+          root: '.'
+          paths:
+            - monorepo_tools_py3_windows_amd64.zip
+  test-windows-standalone-py3:
+    executor:
+      name: vs2019
+    steps:
+      - attach_workspace:
+          at: ./artifacts
+      - run: choco install --no-progress python
+      - run: python ./artifacts/monorepo_tools_py3_windows_amd64.zip --help
+  test-windows-py2:
+    executor:
+      name: vs2019
+    steps:
+      - checkout
+      - run: choco install --no-progress bazel
+      - run: pip install wheel
+      - run: bazel build //...
+      - run: bazel test //...
+      - run: mv bazel-bin/monorepo_tools.zip monorepo_tools_py2_windows_amd64.zip
+      - persist_to_workspace:
+          root: '.'
+          paths:
+            - monorepo_tools_py2_windows_amd64.zip
+  test-windows-standalone-py2:
+    executor:
+      name: vs2019
+    steps:
+      - attach_workspace:
+          at: ./artifacts
+      - run: choco install --no-progress python2
+      - run: python ./artifacts/monorepo_tools_py2_windows_amd64.zip --help
+  test-darwin:
+    macos:
+      xcode: 9.3.0
+    steps:
+      - checkout
+      - run: brew install bazel
+      - run: bazel build //...
+      - run: bazel test //...
+      - run: mv bazel-bin/monorepo_tools.zip monorepo_tools_darwin_amd64.zip
+      - persist_to_workspace:
+          root: '.'
+          paths:
+            - monorepo_tools_darwin_amd64.zip
+  test-darwin-standalone:
+    macos:
+      xcode: 9.3.0
+    steps:
+      - attach_workspace:
+          at: ./artifacts
+      - run: python ./artifacts/monorepo_tools_darwin_amd64.zip --help
+      - run: python2 ./artifacts/monorepo_tools_darwin_amd64.zip --help
+  publish-github-release:
+    docker:
+      - image: cibuilds/github:0.10
+        environment:
+          # https://stackoverflow.com/questions/57828037/cant-attach-circleci-workspace-from-windows-to-linux-due-to-cannot-change-owne
+          TAR_OPTIONS: --no-same-owner
+    steps:
+      - attach_workspace:
+          at: ./artifacts
+      - run:
+          name: "Publish Release on GitHub"
+          command: |
+            VERSION=${CIRCLE_TAG:-${CIRCLE_SHA1}}
+            (
+              cd ./artifacts &&
+              mv monorepo_tools_linux_amd64.zip monorepo_tools_${VERSION}_linux_amd64.zip &&
+              mv monorepo_tools_py2_windows_amd64.zip monorepo_tools_py2_${VERSION}_windows_amd64.zip &&
+              mv monorepo_tools_py3_windows_amd64.zip monorepo_tools_py3_${VERSION}_windows_amd64.zip &&
+              mv monorepo_tools_darwin_amd64.zip monorepo_tools_${VERSION}_darwin_amd64.zip
+            )
+            ghr -draft -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${VERSION} ./artifacts/
+
+workflows:
+  main:
+    jobs:
+      - test-linux:
+          filters:
+            tags:
+              only: /^v.*/
+      - test-linux-standalone:
+          requires:
+            - test-linux
+          filters:
+            tags:
+              only: /^v.*/
+      - test-windows-py3:
+          filters:
+            tags:
+              only: /^v.*/
+      - test-windows-standalone-py3:
+          requires:
+            - test-windows-py3
+          filters:
+            tags:
+              only: /^v.*/
+      - test-windows-py2:
+          filters:
+            tags:
+              only: /^v.*/
+      - test-windows-standalone-py2:
+          requires:
+            - test-windows-py2
+          filters:
+            tags:
+              only: /^v.*/
+      - test-darwin:
+          filters:
+            tags:
+              only: /^v.*/
+      - test-darwin-standalone:
+          requires:
+            - test-darwin
+          filters:
+            tags:
+              only: /^v.*/
+      - publish-github-release:
+          requires:
+            - test-linux-standalone
+            - test-windows-standalone-py2
+            - test-windows-standalone-py3
+            - test-darwin-standalone
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/

.gitignore

@@ -1 +1,3 @@
 /bazel-*
+__pycache__
+*.pyc

BUILD

@@ -0,0 +1,12 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
+load("@py_deps//:requirements.bzl", "requirement")
+
+py_binary(
+    name = "monorepo_tools",
+    srcs = ["cli.py"],
+    main = "cli.py",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//import_into",
+    ],
+)

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Hadrien Chauvin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,116 @@
+# `monorepo-tools`: Monorepo administration
+
+[![CircleCI](https://circleci.com/gh/hchauvin/monorepo-tools/tree/master.svg?style=svg)](https://circleci.com/gh/hchauvin/monorepo-tools/tree/master)
+
+`monorepo-tools` aims at offering a collection of tools to administrate a
+monorepo.  Monorepos have
+[many advantages](https://en.wikipedia.org/wiki/Monorepo) for closed-source systems
+as compared to separate repos, and are a sound evolution or starting point for projects
+in need of large-scale code refactoring, collaboration, and ease of code
+reuse.  A monorepo correctly set up can diminish friction both for
+fledging startups, and for companies maintaining, evolving or migrating
+legacy projects: they can be introduced at all stages of a product's lifecycle.
+
+The tools can be either consumed using a Command-Line Interface (CLI),
+or programmatically.  They are written in Python, packaged in a runnable ZIP
+file, and compatible with Python 2.7 and Python 3.7.  Prepackaged runnable
+ZIP files are available
+[on the release page](https://github.com/hchauvin/monorepo-tools/releases).
+Tests are continuously run on Windows, Linux, and Mac OSX.
+
+Right now, `monorepo-tools` only offers one subcommand, `import`, but other
+commands will follow.  The scope will be vendoring, open sourcing part of
+a monorepo with an OSS-monorepo sync, and related topics.  We plan on
+open-sourcing separately some work on continuous integration and 
+deployment pipelines for monorepos, as CI/CD is out-of-scope for this project.
+Currently only Git is supported as a Version Control System and no plan
+is made to extend support to other VCS such as Mercurial.
+
+## Installation
+
+For CLI use, please go to [the release page](https://github.com/hchauvin/monorepo-tools/releases)
+and download the appropriate ZIP bundle for your platform.  For Windows,
+please make sure that you have Python 2 or Python 3 installed.  On Windows, we
+recommend installing Python using [Chocolatey](https://chocolatey.org) (respectively,
+with `choco install python2` and `choco install python`).  Usage can be queried with:
+
+```bash
+python monorepo_tools.zip --help
+```
+
+For programmatic access, use [bazel](https://bazel.build/) and import
+this project in your workspace.
+
+## `monorepo_tools-import`
+
+```
+usage: monorepo_tools import [-h] --individual_repos INDIVIDUAL_REPOS
+                             --dest_branch DEST_BRANCH --monorepo_path
+                             MONOREPO_PATH
+
+Import individual repos into a monorepo
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --individual_repos INDIVIDUAL_REPOS
+                        Path to python module that exports one function,
+                        individual_repos, that takes the destination branch
+                        name as an argument
+  --dest_branch DEST_BRANCH
+                        The destination branch to import into
+  --monorepo_path MONOREPO_PATH
+                        The local path to the monorepo (it is created if it
+                        does not exist)
+```
+
+Note that incremental update of an existing monorepo is supported, just
+set `--monorepo_path` to a clone.
+
+See [./import_into/individual_repos.py]() for an example for `--individual_repos`.
+
+The strategy for `import` is "merge unrelated history then move": for each
+individual repo, we create in the monorepo a branch that is the result
+of pulling the unrelated history from the requested branch in the individual
+repo.  This history is directly taken from the individual repo, without
+any transformation, meaning that the commit SHA1 are the same, which helps
+for traceability and auditing.  Additionally, because there is no
+transformation, the import is faster than other strategies (see below).
+The files in this branch are moved to the appropriate subdirectory of the
+monorepo (and these moves are committed), then this branch is merged into
+the destination monorepo branch.  This way, `import` introduces two
+additional commits per individual repo and destination branch: a move,
+and a merge.  Additionally, `import` provides the first commit in the
+monorepo branch (with the message "Initial monorepo commit"), onto which
+the individual repos are grafted.  With this strategy, commit history is best
+viewed in date order, not ancestor order.
+
+### Alternatives
+
+While researching `import`, other strategies and tools were looked at.  We
+specifically wanted a tool that would allow the complete import of histories,
+and autonomy of the monorepo from the separate repos.  Therefore, Git
+[submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules)
+and [`git-subrepo`](https://github.com/ingydotnet/git-subrepo) were taken
+out of the picture, as they work by maintaining references to the separate repos.
+
+Next, [Copybara](https://github.com/google/copybara) was
+considered.  However, its iterative filtering strategy is a huge
+performance issue for large separate repos, and it was quickly abandoned,
+as a full migration of the repos we were considering would take Copybara
+many days to perform.
+
+[`git-stitch-repo`](https://metacpan.org/pod/git-stitch-repo)
+was also considered.  It nicely uses `git-fast-import` and `git-fast-export`
+to combine linear histories into one linear history, which could be cleaner 
+than our "merge unrelated history then move" (as it comes with merge "nonlinearities").
+However, we found out that `git-stitch-repo` gave wrong results for nonlinear histories, as
+the commits were sometimes not correctly stitched.  The project, written in Perl, had not
+been maintained for years.  We also decided that Git history
+rewriting was too difficult to get right for the mixed benefits
+of enforcing a linear Git history.  That's why we went back to the
+very simple strategy than ended up being `import` and didn't try to patch
+`git-stitch-repo` instead.
+
+## License
+
+`monorepo-tools` is licensed under [The MIT License](./LICENSE).

WORKSPACE

@@ -0,0 +1,51 @@
+workspace(name = "monorepo_tools")
+
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+# Sanity checks
+
+git_repository(
+    name = "bazel_skylib",
+    remote = "https://github.com/bazelbuild/bazel-skylib",
+    tag = "0.9.0",
+)
+
+load("@bazel_skylib//lib:versions.bzl", "versions")
+
+versions.check("0.29.0")
+
+git_repository(
+    name = "rules_python",
+    commit = "54d1cb35cd54318d59bf38e52df3e628c07d4bbc",
+    remote = "https://github.com/bazelbuild/rules_python.git",
+)
+
+# This call should always be present.
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
+# This one is only needed if you're using the packaging rules.
+load("@rules_python//python:pip.bzl", "pip_repositories")
+
+pip_repositories()
+
+load("@rules_python//python:pip.bzl", "pip_import")
+
+# This rule translates the specified requirements.txt into
+# @my_deps//:requirements.bzl, which itself exposes a pip_install method.
+pip_import(
+    name = "py_deps",
+    requirements = "//:requirements.txt",
+)
+
+# Load the pip_install symbol for my_deps, and create the dependencies'
+# repositories.
+load("@py_deps//:requirements.bzl", "pip_install")
+
+pip_install()
+
+# Linting
+load("//internal:format.bzl", "format_repositories")
+
+format_repositories()