diff --git a/.gitattributes b/.gitattributes index cdc62d047..6e2da3cf3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,4 +15,3 @@ Makefile text # These tests are explicitly given specific line endings test_suite/unix_line_endings.jsonnet text eol=lf test_suite/dos_line_endings.jsonnet text eol=crlf - diff --git a/BUILD b/BUILD index 358acc2be..be8de9ecb 100644 --- a/BUILD +++ b/BUILD @@ -17,21 +17,25 @@ genrule( cc_library( name = "jsonnet-common", srcs = [ - "core/desugaring.cpp", + "core/desugarer.cpp", + "core/formatter.cpp", "core/lexer.cpp", "core/parser.cpp", "core/static_analysis.cpp", + "core/string_utils.cpp", "core/vm.cpp", "core/std.jsonnet.h", ], hdrs = [ "core/ast.h", - "core/desugaring.h", + "core/desugarer.h", + "core/formatter.h", "core/lexer.h", "core/parser.h", "core/state.h", "core/static_analysis.h", "core/static_error.h", + "core/string_utils.h", "core/unicode.h", "core/vm.h", "include/libjsonnet.h", diff --git a/Makefile b/Makefile index 355edaff2..a6d0ebc9a 100644 --- a/Makefile +++ b/Makefile @@ -42,11 +42,13 @@ SHARED_LDFLAGS ?= -shared ################################################################################ LIB_SRC = \ - core/desugaring.cpp \ + core/desugarer.cpp \ + core/formatter.cpp \ core/lexer.cpp \ core/libjsonnet.cpp \ core/parser.cpp \ core/static_analysis.cpp \ + core/string_utils.cpp \ core/vm.cpp LIB_OBJ = $(LIB_SRC:.cpp=.o) @@ -60,12 +62,14 @@ ALL = \ $(LIB_OBJ) ALL_HEADERS = \ core/ast.h \ - core/desugaring.h \ + core/desugarer.h \ + core/formatter.h \ core/lexer.h \ core/parser.h \ core/state.h \ core/static_analysis.h \ core/static_error.h \ + core/string_utils.h \ core/vm.h \ core/std.jsonnet.h \ include/libjsonnet.h @@ -82,7 +86,7 @@ test: jsonnet libjsonnet.so libjsonnet_test_snippet libjsonnet_test_file cd examples ; ./check.sh cd examples/terraform ; ./check.sh cd test_suite ; ./run_tests.sh - cd test_suite ; ./run_unparse_tests.sh + cd test_suite ; ./run_fmt_tests.sh MAKEDEPEND_SRCS = \ cmd/jsonnet.cpp \ @@ -92,7 +96,7 @@ MAKEDEPEND_SRCS = \ depend: makedepend -f- $(LIB_SRC) $(MAKEDEPEND_SRCS) > Makefile.depend -core/desugaring.cpp: core/std.jsonnet.h +core/desugarer.cpp: core/std.jsonnet.h # Object files %.o: %.cpp diff --git a/benchmarks/bench.01.jsonnet b/benchmarks/bench.01.jsonnet index 1881cd6e6..350d97533 100644 --- a/benchmarks/bench.01.jsonnet +++ b/benchmarks/bench.01.jsonnet @@ -18,5 +18,5 @@ local sum(x) = if x == 0 then 0 else - x + sum(x-1); + x + sum(x - 1); sum(300) diff --git a/benchmarks/bench.02.jsonnet b/benchmarks/bench.02.jsonnet index 416ffd01d..068700e93 100644 --- a/benchmarks/bench.02.jsonnet +++ b/benchmarks/bench.02.jsonnet @@ -17,7 +17,7 @@ limitations under the License. local Fib = { n: 1, local outer = self, - r: if self.n <= 1 then 1 else (Fib { n: outer.n - 1}).r + (Fib { n: outer.n - 2}).r + r: if self.n <= 1 then 1 else (Fib { n: outer.n - 1 }).r + (Fib { n: outer.n - 2 }).r, }; (Fib { n: 25 }).r diff --git a/case_studies/fractal/service.jsonnet b/case_studies/fractal/service.jsonnet index 01b2dc6b4..158d54f93 100644 --- a/case_studies/fractal/service.jsonnet +++ b/case_studies/fractal/service.jsonnet @@ -45,7 +45,7 @@ local credentials = import "credentials.jsonnet"; seed_provider: [ { class_name: "org.apache.cassandra.locator.SimpleSeedProvider", - parameters: [ { seeds: std.join(", ", cassandraNodes) } ], + parameters: [{ seeds: std.join(", ", cassandraNodes) }], }, ], }, @@ -100,17 +100,17 @@ local credentials = import "credentials.jsonnet"; // For debugging: local network_debug = ["traceroute", "lsof", "iptraf", "tcpdump", "host", "dnsutils"], - aptPackages +: ["vim", "git", "psmisc", "screen", "strace" ] + network_debug, + aptPackages+: ["vim", "git", "psmisc", "screen", "strace"] + network_debug, }, // Frontend image. "appserv.packer.json": packer.GcpDebianNginxUwsgiFlaskImage + ImageMixin { name: "appserv-v20150430-2145", - module: "main", // Entrypoint in the Python code. - pipPackages +: ["httplib2", "cassandra-driver", "blist"], - uwsgiConf +: { lazy: "true" }, // cassandra-driver does not survive fork() + module: "main", // Entrypoint in the Python code. + pipPackages+: ["httplib2", "cassandra-driver", "blist"], + uwsgiConf+: { lazy: "true" }, // cassandra-driver does not survive fork() // Copy website content and code. - provisioners +: [ + provisioners+: [ packer.File { source: "appserv", destination: "/tmp/", @@ -134,12 +134,12 @@ local credentials = import "credentials.jsonnet"; name: "tilegen-v20150430-2145", module: "mandelbrot_service", - aptPackages +: ["g++", "libpng-dev"], + aptPackages+: ["g++", "libpng-dev"], port: tilegenPort, // Copy the flask handlers and also build the C++ executable. - provisioners +: [ + provisioners+: [ packer.File { source: "tilegen", destination: "/tmp/", @@ -168,7 +168,7 @@ local credentials = import "credentials.jsonnet"; account_file: "service_account_key.json", project: credentials.project, region: "us-central1", - } + }, }, // The deployed resources. @@ -213,12 +213,12 @@ local credentials = import "credentials.jsonnet"; appserv: { name: "appserv", health_checks: ["${google_compute_http_health_check.appserv.name}"], - instances: [ "%s/appserv%d" % [zone(k), k] for k in [1, 2, 3] ], + instances: ["%s/appserv%d" % [zone(k), k] for k in [1, 2, 3]], }, tilegen: { name: "tilegen", health_checks: ["${google_compute_http_health_check.tilegen.name}"], - instances: [ "%s/tilegen%d" % [zone(k), k] for k in [1, 2, 3, 4, 5] ], + instances: ["%s/tilegen%d" % [zone(k), k] for k in [1, 2, 3, 4, 5]], }, }, @@ -234,12 +234,12 @@ local credentials = import "credentials.jsonnet"; name: "tilegen", target: "${google_compute_target_pool.tilegen.self_link}", port_range: tilegenPort, - } + }, }, // Open ports for the various services, to instances (identified by tags) google_compute_firewall: { - local NetworkMixin = { network: "${google_compute_network.fractal.name}", }, + local NetworkMixin = { network: "${google_compute_network.fractal.name}" }, ssh: terraform.GcpFirewallSsh + NetworkMixin { name: "ssh" }, appserv: terraform.GcpFirewallHttp + NetworkMixin { name: "appserv" }, tilegen: terraform.GcpFirewallHttp + NetworkMixin { name: "tilegen", port: tilegenPort }, @@ -249,17 +249,17 @@ local credentials = import "credentials.jsonnet"; // All our instances share this configuration. local FractalInstance(zone_hash) = terraform.GcpInstance { - network_interface: [super.network_interface[0] + {network: "${google_compute_network.fractal.name}"} ], - tags +: ["fractal"], + network_interface: [super.network_interface[0] { network: "${google_compute_network.fractal.name}" }], + tags+: ["fractal"], zone: zone(zone_hash), - scopes +: ["devstorage.full_control"] + scopes+: ["devstorage.full_control"], }, // The various kinds of Cassandra instances all share this basic configuration. local CassandraInstance(zone_hash) = FractalInstance(zone_hash) { image: "cassandra-v20150430-2145", machine_type: "n1-standard-1", - tags +: ["fractal-db", "cassandra-server"], + tags+: ["fractal-db", "cassandra-server"], user:: cassandraUser, userPass:: credentials.cassandraUserPass, rootPass:: credentials.cassandraRootPass, @@ -279,8 +279,8 @@ local credentials = import "credentials.jsonnet"; database_user: cassandraUser, database_pass: credentials.cassandraUserPass, }, - tags +: ["fractal-appserv", "http-server"], - startup_script +: [self.addFile(self.conf, "/var/www/conf.json")], + tags+: ["fractal-appserv", "http-server"], + startup_script+: [self.addFile(self.conf, "/var/www/conf.json")], } for k in [1, 2, 3] @@ -326,8 +326,8 @@ local credentials = import "credentials.jsonnet"; ["tilegen" + k]: FractalInstance(k) { name: "tilegen" + k, image: "tilegen-v20150430-2145", - tags +: ["fractal-tilegen", "http-server"], - startup_script +: [self.addFile(ApplicationConf, "/var/www/conf.json")], + tags+: ["fractal-tilegen", "http-server"], + startup_script+: [self.addFile(ApplicationConf, "/var/www/conf.json")], } for k in [1, 2, 3, 4] }, @@ -337,7 +337,7 @@ local credentials = import "credentials.jsonnet"; name: "fractaldemo-com", dns_name: credentials.dnsPrefix + "fractaldemo.com.", description: "Fractal Demo DNS Zone", - } + }, }, local instances = self.google_compute_instance, @@ -374,14 +374,14 @@ local credentials = import "credentials.jsonnet"; ttl: 300, rrdatas: ["${google_compute_address.appserv.address}"], }, - } + }, }, output: { - "frontend": { value: "${google_compute_address.appserv.address}" }, + frontend: { value: "${google_compute_address.appserv.address}" }, }, - } // terraform.tf + }, // terraform.tf } diff --git a/case_studies/kubernetes/example.jsonnet b/case_studies/kubernetes/example.jsonnet index dac2542d4..d2ed808f4 100644 --- a/case_studies/kubernetes/example.jsonnet +++ b/case_studies/kubernetes/example.jsonnet @@ -17,7 +17,7 @@ local Kube = import "libkube.jsonnet"; { # The port that this service should serve on. - redis_port:: 6379, + redis_port:: 6379, # Change this to your project ID. project_id:: "cooltool-1009", @@ -38,8 +38,8 @@ local Kube = import "libkube.jsonnet"; template: { metadata: { labels: { - name: "twitter-stream" - } + name: "twitter-stream", + }, }, spec: { containers: [ @@ -55,11 +55,11 @@ local Kube = import "libkube.jsonnet"; ACCESSTOKENSEC: $.twitter_access_token_sec, TWSTREAMMODE: "sample", }), - } - ] - } - } - } + }, + ], + }, + }, + }, }, "redis-master-service.new.yaml": Kube.v1.Service("redis-master") { metadata+: { @@ -72,12 +72,12 @@ local Kube = import "libkube.jsonnet"; # You don't need to specify the targetPort if it is the same as the port, # though here we include it anyway, to show the syntax. targetPort: $.redis_port, - } + }, ], selector: { name: "redis-master", - } - } + }, + }, }, "redis-master.new.yaml": Kube.v1.ReplicationController("redis-master") { spec: { @@ -86,7 +86,7 @@ local Kube = import "libkube.jsonnet"; metadata: { labels: { name: "redis-master", - } + }, }, spec: { containers: [ @@ -103,11 +103,11 @@ local Kube = import "libkube.jsonnet"; name: "collectd", image: "gcr.io/%s/collectd-redis:latest" % $.project_id, ports: [], - } - ] - } - } - } + }, + ], + }, + }, + }, }, "bigquery-controller.new.yaml": Kube.v1.ReplicationController("bigquery-controller") { spec: { @@ -116,7 +116,7 @@ local Kube = import "libkube.jsonnet"; metadata: { labels: { name: "bigquery-controller", - } + }, }, spec: { containers: [ @@ -131,10 +131,10 @@ local Kube = import "libkube.jsonnet"; BQ_DATASET: $.bq_dataset, BQ_TABLE: $.bq_table, }), - } - ] - } - } - } - } + }, + ], + }, + }, + }, + }, } diff --git a/case_studies/kubernetes/libkube.jsonnet b/case_studies/kubernetes/libkube.jsonnet index 53d3779d2..790707fa6 100644 --- a/case_studies/kubernetes/libkube.jsonnet +++ b/case_studies/kubernetes/libkube.jsonnet @@ -35,11 +35,11 @@ Service(name): ApiVersion + Metadata(name) { kind: "Service", - } + }, }, pair_list_ex(tab, kfield, vfield):: - [{[kfield]: k, [vfield]: tab[k]} for k in std.objectFields(tab)], + [{ [kfield]: k, [vfield]: tab[k] } for k in std.objectFields(tab)], pair_list(tab):: self.pair_list_ex(tab, "name", "value"), diff --git a/case_studies/lib/cassandra.jsonnet b/case_studies/lib/cassandra.jsonnet index c71c65e34..2c39dfe68 100644 --- a/case_studies/lib/cassandra.jsonnet +++ b/case_studies/lib/cassandra.jsonnet @@ -30,7 +30,7 @@ local packer = import "packer.jsonnet"; client_encryption_options: { enabled: false, keystore: "conf/.keystore", - keystore_password: "cassandra" + keystore_password: "cassandra", }, cluster_name: "Unnamed Cluster", column_index_size_in_kb: 64, @@ -84,15 +84,15 @@ local packer = import "packer.jsonnet"; seed_provider: [ { class_name: "org.apache.cassandra.locator.SimpleSeedProvider", - parameters: [ { seeds: "127.0.0.1" } ], - } + parameters: [{ seeds: "127.0.0.1" }], + }, ], server_encryption_options: { internode_encryption: "none", keystore: "conf/.keystore", keystore_password: "cassandra", truststore: "conf/.truststore", - truststore_password: "cassandra" + truststore_password: "cassandra", }, snapshot_before_compaction: false, ssl_storage_port: 7001, @@ -106,7 +106,7 @@ local packer = import "packer.jsonnet"; trickle_fsync: false, trickle_fsync_interval_in_kb: 10240, truncate_request_timeout_in_ms: 60000, - write_request_timeout_in_ms: 2000 + write_request_timeout_in_ms: 2000, }, // Some firewall resources to open up Cassandra ports. @@ -118,7 +118,7 @@ local packer = import "packer.jsonnet"; // From the Internet to these ports. target_tags: [self.cassandraTag], }, - GcpFirewallGossip:: { + GcpFirewallGossip:: { cassandraTag:: "cassandra-server", source_ranges: ["0.0.0.0/0"], network: error "cassandra.GcpFirewallGossip must have field: network", @@ -129,40 +129,40 @@ local packer = import "packer.jsonnet"; }, // Sets the root password to something, while the server is listening only on localhost. - GcpDebianImage: packer.GcpDebianImage { + GcpDebianImage: packer.GcpDebianImage { local image = self, rootPassword:: error "GcpDebianCassandraPrimedImage: must have field: rootPassword", clusterName:: error "GcpDebianCassandraPrimedImage: must have field: clusterName", - aptKeyUrls +: ["https://www.apache.org/dist/cassandra/KEYS"], - aptRepoLines +: { + aptKeyUrls+: ["https://www.apache.org/dist/cassandra/KEYS"], + aptRepoLines+: { cassandra: "deb http://www.apache.org/dist/cassandra/debian 21x main", }, - aptPackages +: ["cassandra"], + aptPackages+: ["cassandra"], conf:: cassandra.conf { authenticator: "PasswordAuthenticator", cluster_name: image.clusterName, }, - provisioners +: [ + provisioners+: [ packer.RootShell { - inline: [ + inline: [ // Shut it down "/etc/init.d/cassandra stop", // Remove junk from unconfigured startup "rm -rfv /var/lib/cassandra/*", // Enable authentication local dest = "/etc/cassandra/cassandra.yaml"; - "echo %s > %s" % [std.escapeStringBash("" + image.conf), dest], + "echo %s > %s" % [std.escapeStringBash("" + image.conf), dest], // Start it up again "/etc/init.d/cassandra start", // Wait for it to be ready cassandra.waitForCqlsh("cassandra", "cassandra", "localhost"), // Set root password local cql = "ALTER USER cassandra WITH PASSWORD '%s';" % image.rootPassword; - "echo %s | cqlsh -u cassandra -p cassandra" % std.escapeStringBash(cql), + "echo %s | cqlsh -u cassandra -p cassandra" % std.escapeStringBash(cql), ], }, ], @@ -181,19 +181,19 @@ local packer = import "packer.jsonnet"; // given CQL script and configuration. GcpStarterMixin: { - startup_script +: [ + startup_script+: [ // Wait for the misconfigured cassandra to start up. cassandra.waitForCqlsh("cassandra", self.rootPass, "localhost"), // Set up system_auth replication level "echo %s | cqlsh -u cassandra -p %s localhost" - % [std.escapeStringBash("ALTER KEYSPACE system_auth WITH REPLICATION = %s;" - % self.authReplication), - self.rootPass], + % [std.escapeStringBash("ALTER KEYSPACE system_auth WITH REPLICATION = %s;" + % self.authReplication), + self.rootPass], // Drop in the correct configuration. "echo %s > %s" - % [std.escapeStringBash("" + self.conf), "/etc/cassandra/cassandra.yaml"], + % [std.escapeStringBash("" + self.conf), "/etc/cassandra/cassandra.yaml"], // Restart with new configuration. "/etc/init.d/cassandra restart", @@ -203,7 +203,7 @@ local packer = import "packer.jsonnet"; // Set up users, empty tables, etc. "echo %s | cqlsh -u cassandra -p %s $HOSTNAME" - % [std.escapeStringBash(std.lines(self.initCql)), self.rootPass], + % [std.escapeStringBash(std.lines(self.initCql)), self.rootPass], ], }, @@ -212,7 +212,7 @@ local packer = import "packer.jsonnet"; // GcpDebianImage. It adds commands to the startup script that bootstrap Cassandra by wiping // its slate clean and allowing it to join an existing cluster. GcpTopUpMixin: { - startup_script +: [ + startup_script+: [ // Wait for the misconfigured cassandra to start up. cassandra.waitForCqlsh("cassandra", self.rootPass, "localhost"), @@ -224,7 +224,7 @@ local packer = import "packer.jsonnet"; // Drop in the correct configuration. "echo %s > %s" - % [std.escapeStringBash("" + self.conf), "/etc/cassandra/cassandra.yaml"], + % [std.escapeStringBash("" + self.conf), "/etc/cassandra/cassandra.yaml"], // Start it up again. "/etc/init.d/cassandra start", diff --git a/case_studies/lib/packer.jsonnet b/case_studies/lib/packer.jsonnet index 2589a0395..1f23eb557 100644 --- a/case_studies/lib/packer.jsonnet +++ b/case_studies/lib/packer.jsonnet @@ -19,13 +19,13 @@ limitations under the License. // A Packer shell provisioner to run something as root. RootShell:: { - type: "shell", - execute_command: "{{ .Vars }} sudo -E /bin/bash '{{ .Path }}'", + type: "shell", + execute_command: "{{ .Vars }} sudo -E /bin/bash '{{ .Path }}'", }, // A Packer shell provisioner to run something as the current user. Shell:: { - type: "shell", + type: "shell", }, // A Packer file provisioner. @@ -89,7 +89,7 @@ limitations under the License. // packages are given as an array of strings. Pip:: packer.RootShell { packages:: error "Pip provisioner must have field: packages", - inline: [ "pip install " + std.join(" ", self.packages) ], + inline: ["pip install " + std.join(" ", self.packages)], }, // A Packer provisioner to install Apt packages. This provisioner can be configured with @@ -102,7 +102,7 @@ limitations under the License. // { foo: "..." } will add a foo.list fetched from the given URL. repoUrls:: {}, - keyCommands:: [ "curl --silent %s | apt-key add -" % [url] for url in self.keyUrls], + keyCommands:: ["curl --silent %s | apt-key add -" % [url] for url in self.keyUrls], local dir = "/etc/apt/sources.list.d", local repoLineCommands = ["echo \"%s\" > %s/%s.list" % [self.repoLines[k], dir, k] @@ -112,11 +112,11 @@ limitations under the License. repoCommands:: repoLineCommands + repoUrlCommands, local opts = "-o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold", - installCommands:: [ "apt-get -qq -y %s install %s" % [opts, std.join(" ", self.packages) ]], + installCommands:: ["apt-get -qq -y %s install %s" % [opts, std.join(" ", self.packages)]], - environment_vars: [ "DEBIAN_FRONTEND=noninteractive" ], + environment_vars: ["DEBIAN_FRONTEND=noninteractive"], inline: self.repoCommands + self.keyCommands - + [ "apt-get -qq -y update" ] + self.installCommands, + + ["apt-get -qq -y update"] + self.installCommands, }, // TODO(dcunnin): yum /* @@ -145,7 +145,7 @@ limitations under the License. local img = self, builder:: { - type: "googlecompute", + type: "googlecompute", name: img.name, image_name: "%s" % [self.name], @@ -153,18 +153,18 @@ limitations under the License. // Project & authentication project_id: img.project_id, - account_file: img.account_file, + account_file: img.account_file, // Instance mechanics machine_type: "n1-standard-1", // Multicore probably doesn't provide any benefit source_image: img.source_image, instance_name: "packer-" + self.name, - zone: "us-central1-a", - ssh_username: std.extVar("USER"), + zone: "us-central1-a", + ssh_username: std.extVar("USER"), [if std.objectHas(img, "disk_size") then "disk_size"]: img.disk_size, }, - builders: [ self.builder ], + builders: [self.builder], provisioners: [], }, @@ -187,10 +187,10 @@ limitations under the License. local apt_provisioners = if std.length(pip_pkgs) == 0 - && std.length(self.aptKeyUrls) == 0 + && std.length(self.aptKeyUrls) == 0 && std.length(self.aptRepos) == 0 then [ ] else [ - packer.Apt { + packer.Apt { packages: pip_pkgs, keyUrls: image.aptKeyUrls, repoLines: image.aptRepoLines, @@ -201,11 +201,11 @@ limitations under the License. local pip_provisioners = if std.length(self.pipPackages) == 0 then [ ] else [ - packer.Pip { packages: image.pipPackages, }, + packer.Pip { packages: image.pipPackages }, ], - provisioners +: apt_provisioners + pip_provisioners, - + provisioners+: apt_provisioners + pip_provisioners, + }, // A template for building Nginx/uwsgi/flask based application servers. The uwsgi configuration @@ -220,8 +220,8 @@ limitations under the License. port:: 80, uwsgiSocket:: "/var/www/uwsgi.sock", - aptPackages +: ["nginx", "python-dev"], - pipPackages +: ["flask", "uwsgi"], + aptPackages+: ["nginx", "python-dev"], + pipPackages+: ["flask", "uwsgi"], uwsgiConf:: { chdir: "/var/www", @@ -248,8 +248,8 @@ limitations under the License. "}", ], - provisioners +: [ - packer.RootShell { inline: ["rm /etc/nginx/sites-enabled/default"], }, + provisioners+: [ + packer.RootShell { inline: ["rm /etc/nginx/sites-enabled/default"] }, packer.EnsureFile { destination: "/etc/nginx/conf.d/frontend_nginx.conf", content: std.lines(image.nginxConf), @@ -258,35 +258,35 @@ limitations under the License. destination: "/etc/uwsgi/vassals/uwsgi.ini", content: std.manifestIni({ sections: { - uwsgi: image.uwsgiConf - } - }) + uwsgi: image.uwsgiConf, + }, + }), }, packer.EnsureFile { destination: "/etc/cron.d/emperor", content: "@reboot root /usr/local/bin/uwsgi --master --emperor /etc/uwsgi/vassals " + "--daemonize /var/log/uwsgi/emperor.log --pidfile /var/run/uwsgi.pid " - + "--die-on-term --uid www-data --gid www-data\n" + + "--die-on-term --uid www-data --gid www-data\n", }, - packer.EnsureDir { dir: "/var/log/uwsgi", user: "www-data", }, - packer.EnsureDir { dir: "/var/www", user: "www-data", }, + packer.EnsureDir { dir: "/var/log/uwsgi", user: "www-data" }, + packer.EnsureDir { dir: "/var/www", user: "www-data" }, ], }, // A template to help build PostgreSQL images (experimental). - GcpDebianPostgresqlImage: packer.GcpDebianImage { + GcpDebianPostgresqlImage: packer.GcpDebianImage { local image = self, rootPassword:: error "GcpDebianPostgresqlImage: must have field: rootPassword", - aptPackages +: ["postgresql", "postgresql-contrib"], + aptPackages+: ["postgresql", "postgresql-contrib"], initSql:: [ "ALTER USER POSTGRES WITH PASSWORD '%s';" % image.rootPassword, ], - provisioners +: [ + provisioners+: [ packer.RootShell { inline: [ "echo %s | sudo -u postgres psql" % std.escapeStringBash(std.lines(image.initSql)), ] }, @@ -310,7 +310,7 @@ limitations under the License. "lc_numeric = 'en_US.UTF-8'", "lc_time = 'en_US.UTF-8'", "default_text_search_config = 'pg_catalog.english'", - ]) + ]), }, packer.EnsureFile { destination: "/etc/postgresql/9.1/main/pg_hba.conf", @@ -318,7 +318,7 @@ limitations under the License. "local all all md5", "host all all 255.255.255.255/0 md5", "host all all ::1/128 md5", - ]) + ]), }, ], }, @@ -329,11 +329,11 @@ limitations under the License. rootPassword:: error "GcpDebianMysqlImage: must have field: rootPassword", - aptPackages +: ["mysql-server"], + aptPackages+: ["mysql-server"], initSql:: [], - provisioners +: [ + provisioners+: [ packer.RootShell { inline: [ "mysqladmin -u root password '%s'" % std.escapeStringBash(image.rootPassword), "echo %s | mysql -u root --password=%s" @@ -346,11 +346,10 @@ limitations under the License. sections: { mysqld: { "bind-address": "0.0.0.0", - } + }, }, }), }, ], }, } - diff --git a/case_studies/lib/terraform.jsonnet b/case_studies/lib/terraform.jsonnet index fde284605..998dec3eb 100644 --- a/case_studies/lib/terraform.jsonnet +++ b/case_studies/lib/terraform.jsonnet @@ -30,7 +30,7 @@ limitations under the License. service_account: [ { scopes: ["https://www.googleapis.com/auth/" + s for s in instance.scopes], - } + }, ], machine_type: "f1-micro", @@ -53,10 +53,10 @@ limitations under the License. network_interface: [{ network: "default", access_config: if instance.address != null then [ - { nat_ip: instance.address } + { nat_ip: instance.address }, ] else [ - { } - ] + {}, + ], }], }, diff --git a/case_studies/micro_fractal/fractal.jsonnet b/case_studies/micro_fractal/fractal.jsonnet index 123b3f2eb..a54877362 100644 --- a/case_studies/micro_fractal/fractal.jsonnet +++ b/case_studies/micro_fractal/fractal.jsonnet @@ -26,7 +26,7 @@ local web_solutions = import "mmlib/v0.1.1/web/solutions.jsonnet"; BaseInstanceMixin:: { StandardRootImage+: { local network_debug = ["traceroute", "lsof", "iptraf", "tcpdump", "host", "dnsutils"], - aptPackages +: ["vim", "git", "psmisc", "screen", "strace"] + network_debug, + aptPackages+: ["vim", "git", "psmisc", "screen", "strace"] + network_debug, }, }, @@ -55,7 +55,7 @@ local web_solutions = import "mmlib/v0.1.1/web/solutions.jsonnet"; Instance+: $.BaseInstanceMixin { StandardRootImage+: { - pipPackages +: ["httplib2", "cassandra-driver", "blist"], + pipPackages+: ["httplib2", "cassandra-driver", "blist"], }, local version = self, conf:: app.ApplicationConf { @@ -67,7 +67,7 @@ local web_solutions = import "mmlib/v0.1.1/web/solutions.jsonnet"; tilegen_port: app.tilegenPort, }, module: "main", // Entrypoint in the Python code. - uwsgiConf +: { lazy: "true" }, // cassandra-driver does not survive fork() + uwsgiConf+: { lazy: "true" }, // cassandra-driver does not survive fork() // Copy website content and code. httpContentCmds+: [ "echo '%s tilegen' >> /etc/hosts" % app.tilegen.refAddress("fractal-tilegen"), @@ -87,7 +87,7 @@ local web_solutions = import "mmlib/v0.1.1/web/solutions.jsonnet"; httpPort: app.tilegenPort, Instance+: $.BaseInstanceMixin { StandardRootImage+: { - aptPackages +: ["g++", "libpng-dev"], + aptPackages+: ["g++", "libpng-dev"], }, local version = self, srcDir:: "tilegen3", @@ -120,7 +120,7 @@ local web_solutions = import "mmlib/v0.1.1/web/solutions.jsonnet"; seed_provider: [ { class_name: "org.apache.cassandra.locator.SimpleSeedProvider", - parameters: [ { seeds: std.join(", ", app.cassandraNodes) } ], + parameters: [{ seeds: std.join(", ", app.cassandraNodes) }], }, ], }, @@ -153,8 +153,8 @@ local web_solutions = import "mmlib/v0.1.1/web/solutions.jsonnet"; ], }, - StarterNode +: $.BaseInstanceMixin + self.FractalStarterMixin, - TopUpNode +: $.BaseInstanceMixin, + StarterNode+: $.BaseInstanceMixin + self.FractalStarterMixin, + TopUpNode+: $.BaseInstanceMixin, nodes: { n1: db.StarterNode { diff --git a/case_studies/micromanage/examples/hello_world_amazon.jsonnet b/case_studies/micromanage/examples/hello_world_amazon.jsonnet index 92ba36d2c..499bce7fd 100644 --- a/case_studies/micromanage/examples/hello_world_amazon.jsonnet +++ b/case_studies/micromanage/examples/hello_world_amazon.jsonnet @@ -33,11 +33,11 @@ local cmd = import "mmlib/v0.1.1/cmd/cmd.jsonnet"; mynetwork: service_amazon.Network { subnets: { "us-west-1c": "10.0.0.0/24", - } + }, }, // Simple case -- one machine serving this Python script. -/* + /* helloworld: service_amazon.SingleInstance + web.HttpSingleInstance + web_solutions.DebianFlaskHttpService { zone: "us-west-1c", @@ -45,14 +45,14 @@ local cmd = import "mmlib/v0.1.1/cmd/cmd.jsonnet"; uwsgiModuleContent: ||| import flask import socket - app = flask.Flask(__name__) - @app.route('/') + app = flask.Flask(__name__) + @app.route('/') def hello_world(): return 'Hello from %s!' % socket.gethostname() |||, networkName: "mynetwork", }, -*/ + */ // For production -- allows canarying changes, also use a dns zone helloworld2: service_amazon.Cluster3 + web.HttpService3 @@ -97,7 +97,7 @@ local cmd = import "mmlib/v0.1.1/cmd/cmd.jsonnet"; // dnsZoneName: "dns", }, -/* + /* dns: service_amazon.DnsZone { local service = self, dnsName: "hw.example.com.", @@ -110,6 +110,6 @@ local cmd = import "mmlib/v0.1.1/cmd/cmd.jsonnet"; zoneName: "dns", target: "helloworld2", }, -*/ + */ } diff --git a/case_studies/micromanage/examples/hello_world_google.jsonnet b/case_studies/micromanage/examples/hello_world_google.jsonnet index 88bd2b963..44f603c84 100644 --- a/case_studies/micromanage/examples/hello_world_google.jsonnet +++ b/case_studies/micromanage/examples/hello_world_google.jsonnet @@ -25,7 +25,7 @@ local web_solutions = import "mmlib/v0.1.1/web/solutions.jsonnet"; region: "us-central1", // Maybe change this. // Download this file from the developers console. serviceAccount: import "service_account.json", - sshUser: "yourusername" // Change this. + sshUser: "yourusername", // Change this. }, }, @@ -86,7 +86,7 @@ local web_solutions = import "mmlib/v0.1.1/web/solutions.jsonnet"; // dnsZoneName: "dns", }, -/* + /* dns: service_google.DnsZone { local service = self, dnsName: "hw.example.com.", @@ -99,6 +99,6 @@ local web_solutions = import "mmlib/v0.1.1/web/solutions.jsonnet"; zoneName: "dns", target: "helloworld2", }, -*/ + */ } diff --git a/case_studies/micromanage/lib/mmlib/v0.1.0/cmd/apt.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.0/cmd/apt.jsonnet index 7c5978783..e9c5a296a 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.0/cmd/apt.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.0/cmd/apt.jsonnet @@ -11,22 +11,22 @@ local pip = import "pip.jsonnet"; aptRepoLines:: {}, // { foo: "..." } will add a foo.list fetched from the given URL. aptRepoUrls:: {}, - aptPackages :: [], + aptPackages:: [], local repoLineCommands = ["echo \"%s\" > %s/%s.list" % [self.aptRepoLines[k], self.aptSourcesListDir, k] for k in std.objectFields(self.aptRepoLines)], local repoUrlCommands = ["curl -o %s/%s.list %s" % [self.aptSourcesListDir, k, self.aptRepoUrls[k]] for k in std.objectFields(self.aptRepoUrls)], - local keyCommands = [ "curl --silent %s | apt-key add -" % [url] for url in self.aptKeyUrls], + local keyCommands = ["curl --silent %s | apt-key add -" % [url] for url in self.aptKeyUrls], local env = "DEBIAN_FRONTEND=noninteractive", local opts = "-o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold", - local updateCommands = [ "apt-get -qq -y update" ], - local installCommands = [ "%s apt-get -qq -y %s install %s" % [env, opts, std.join(" ", self.aptPackages) ]], + local updateCommands = ["apt-get -qq -y update"], + local installCommands = ["%s apt-get -qq -y %s install %s" % [env, opts, std.join(" ", self.aptPackages)]], cmds+: repoLineCommands + repoUrlCommands + keyCommands - + updateCommands + installCommands, + + updateCommands + installCommands, }, PipMixin:: pip.Mixin { diff --git a/case_studies/micromanage/lib/mmlib/v0.1.0/cmd/pip.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.0/cmd/pip.jsonnet index 7360a1562..9df850e9f 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.0/cmd/pip.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.0/cmd/pip.jsonnet @@ -5,7 +5,7 @@ [] else ["pip install " + std.join(" ", self.pipPackages)], - pipPackages :: [], + pipPackages:: [], }, } diff --git a/case_studies/micromanage/lib/mmlib/v0.1.0/db/cassandra.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.0/db/cassandra.jsonnet index 23473006b..508e42a15 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.0/db/cassandra.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.0/db/cassandra.jsonnet @@ -18,7 +18,7 @@ local service_amazon = import "../service/amazon.jsonnet"; client_encryption_options: { enabled: false, keystore: "conf/.keystore", - keystore_password: "cassandra" + keystore_password: "cassandra", }, cluster_name: "Unnamed Cluster", column_index_size_in_kb: 64, @@ -72,15 +72,15 @@ local service_amazon = import "../service/amazon.jsonnet"; seed_provider: [ { class_name: "org.apache.cassandra.locator.SimpleSeedProvider", - parameters: [ { seeds: "127.0.0.1" } ], - } + parameters: [{ seeds: "127.0.0.1" }], + }, ], server_encryption_options: { internode_encryption: "none", keystore: "conf/.keystore", keystore_password: "cassandra", truststore: "conf/.truststore", - truststore_password: "cassandra" + truststore_password: "cassandra", }, snapshot_before_compaction: false, ssl_storage_port: 7001, @@ -94,7 +94,7 @@ local service_amazon = import "../service/amazon.jsonnet"; trickle_fsync: false, trickle_fsync_interval_in_kb: 10240, truncate_request_timeout_in_ms: 60000, - write_request_timeout_in_ms: 2000 + write_request_timeout_in_ms: 2000, }, @@ -111,8 +111,6 @@ local service_amazon = import "../service/amazon.jsonnet"; tcpFirewallPorts:: self.gossipPorts + self.otherPorts, - - StandardNode:: service_google.StandardInstance { conf:: service.defaultConf { authenticator: "PasswordAuthenticator", @@ -210,8 +208,8 @@ local service_amazon = import "../service/amazon.jsonnet"; obj: "org.apache.cassandra.db:type=StorageProxy", attr: ["RecentReadLatencyMicros", "RecentWriteLatencyMicros", "RecentRangeLatencyMicros", "HintsInProgress"], - } - ] + }, + ], }, enableLogging: true, @@ -235,14 +233,14 @@ local service_amazon = import "../service/amazon.jsonnet"; "rm -rfv /var/log/cassandra/*", // Enable authentication local dest = "/etc/cassandra/cassandra.yaml"; - "echo %s > %s" % [std.escapeStringBash("" + bootstrapConf), dest], + "echo %s > %s" % [std.escapeStringBash("" + bootstrapConf), dest], // Start it up again (for some reason 'start' does not do anything...) "/etc/init.d/cassandra restart", // Wait for it to be ready wait_for_cqlsh("cassandra", "cassandra", "localhost"), // Set root password local cql = "ALTER USER cassandra WITH PASSWORD '%s';" % service.rootPassword; - "echo %s | cqlsh -u cassandra -p cassandra" % std.escapeStringBash(cql), + "echo %s | cqlsh -u cassandra -p cassandra" % std.escapeStringBash(cql), ], }, @@ -260,13 +258,13 @@ local service_amazon = import "../service/amazon.jsonnet"; // Set up system_auth replication level "echo %s | cqlsh -u cassandra -p %s localhost" - % [std.escapeStringBash("ALTER KEYSPACE system_auth WITH REPLICATION = %s;" - % node.initAuthReplication), - service.rootPassword], + % [std.escapeStringBash("ALTER KEYSPACE system_auth WITH REPLICATION = %s;" + % node.initAuthReplication), + service.rootPassword], // Drop in the correct configuration. "echo %s > %s" - % [std.escapeStringBash("" + node.conf), "/etc/cassandra/cassandra.yaml"], + % [std.escapeStringBash("" + node.conf), "/etc/cassandra/cassandra.yaml"], // Restart with new configuration. "/etc/init.d/cassandra restart", @@ -276,8 +274,8 @@ local service_amazon = import "../service/amazon.jsonnet"; // Set up users, empty tables, etc. "echo %s | cqlsh -u cassandra -p %s $HOSTNAME" - % [std.escapeStringBash(std.lines(node.initCql)), service.rootPassword], - ] + % [std.escapeStringBash(std.lines(node.initCql)), service.rootPassword], + ], }, TopUpNode:: self.StandardNode { @@ -294,12 +292,12 @@ local service_amazon = import "../service/amazon.jsonnet"; // Drop in the correct configuration. "echo %s > %s" - % [std.escapeStringBash("" + node.conf), "/etc/cassandra/cassandra.yaml"], + % [std.escapeStringBash("" + node.conf), "/etc/cassandra/cassandra.yaml"], // Start it up again. "/etc/init.d/cassandra restart", - ] + ], }, nodes:: {}, @@ -320,7 +318,7 @@ local service_amazon = import "../service/amazon.jsonnet"; { disk: "${google_compute_disk.${-}-%s.name}" % n, auto_delete: false, - } + }, ], } for n in std.objectFields(service.nodes) @@ -333,11 +331,11 @@ local service_amazon = import "../service/amazon.jsonnet"; allow: [ { protocol: "tcp", - ports: [std.toString(p) for p in service.tcpFirewallPorts] - } + ports: [std.toString(p) for p in service.tcpFirewallPorts], + }, ], target_tags: [service.clusterName], - } + }, }, }, }, diff --git a/case_studies/micromanage/lib/mmlib/v0.1.0/service/amazon.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.0/service/amazon.jsonnet index 3aaad2775..085a3a9cb 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.0/service/amazon.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.0/service/amazon.jsonnet @@ -38,7 +38,7 @@ local base = import "base.jsonnet"; service.versions[vname] { name: "${-}-%s-%d" % [vname, i], zone: self.zones[i % std.length(self.zones)], - tags+: [vname, "index-%d" % i] + tags+: [vname, "index-%d" % i], } else error "Undefined version: %s" % vname @@ -54,7 +54,7 @@ local base = import "base.jsonnet"; ]), infrastructure+: { -/* + /* google_compute_firewall: { "${-}": { name: "${-}", @@ -64,7 +64,7 @@ local base = import "base.jsonnet"; target_tags: ["${-}"], } }, -*/ + */ aws_instance: instances, aws_elb: { diff --git a/case_studies/micromanage/lib/mmlib/v0.1.0/service/google.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.0/service/google.jsonnet index d8cc79f67..250ea416f 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.0/service/google.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.0/service/google.jsonnet @@ -36,17 +36,17 @@ local apt = import "../cmd/apt.jsonnet"; network: instance.networkName, access_config: { }, - }, + }, service_account: [ { scopes: ["https://www.googleapis.com/auth/" + s for s in instance.scopes], - } - ], + }, + ], disk: [ - { + { image: instance.StandardRootImage, - } - ], + }, + ], enableLogging:: false, enableMonitoring:: false, @@ -55,19 +55,19 @@ local apt = import "../cmd/apt.jsonnet"; jmxPort:: 9012, jmxHotspotConfig:: { host: instance.jmxHost, - port : instance.jmxPort, - numQueryThreads : 2, + port: instance.jmxPort, + numQueryThreads: 2, SdQuery:: { outputWriters: [ - { - "@class" : "com.googlecode.jmxtrans.model.output.StackdriverWriter", - settings: { - token: "STACKDRIVER_API_KEY", - detectInstance: "GCE", - url: "https://jmx-gateway.google.stackdriver.com/v1/custom" - } - } - ], + { + "@class": "com.googlecode.jmxtrans.model.output.StackdriverWriter", + settings: { + token: "STACKDRIVER_API_KEY", + detectInstance: "GCE", + url: "https://jmx-gateway.google.stackdriver.com/v1/custom", + }, + }, + ], }, queries: [ self.SdQuery { @@ -86,8 +86,8 @@ local apt = import "../cmd/apt.jsonnet"; attr: ["Uptime"], }, self.SdQuery { - resultAlias : "jvm.localhost.os", - obj : "java.lang:type=OperatingSystem", + resultAlias: "jvm.localhost.os", + obj: "java.lang:type=OperatingSystem", attr: [ "CommittedVirtualMemorySize", "FreePhysicalMemorySize", @@ -95,20 +95,20 @@ local apt = import "../cmd/apt.jsonnet"; "OpenFileDescriptorCount", "ProcessCpuTime", "SystemLoadAverage", - ] + ], }, self.SdQuery { resultAlias: "jvm.localhost.gc", obj: "java.lang:type=GarbageCollector,name=*", attr: ["CollectionCount", "CollectionTime"], - } + }, ], }, jmxLocalhostConfig:: instance.jmxHotspotConfig, jmxConfig:: { servers: [ instance.jmxLocalhostConfig, - ] + ], }, MonitoringLoggingImageMixin:: { @@ -148,8 +148,8 @@ local apt = import "../cmd/apt.jsonnet"; local service = self, infrastructure+: if self.dnsZone == null then { } else { - local instances = if std.objectHas(self, "google_compute_instance") then self.google_compute_instance else { }, - local addresses = if std.objectHas(self, "google_compute_address") then self.google_compute_address else { }, + local instances = if std.objectHas(self, "google_compute_instance") then self.google_compute_instance else {}, + local addresses = if std.objectHas(self, "google_compute_address") then self.google_compute_address else {}, local DnsRecord = { managed_zone: service.dnsZone.refName(service.dnsZoneName), type: "A", @@ -189,16 +189,16 @@ local apt = import "../cmd/apt.jsonnet"; infrastructure+: { google_compute_address: { - "${-}": { name: "${-}" } + "${-}": { name: "${-}" }, }, google_compute_firewall: { "${-}": { name: "${-}", source_ranges: ["0.0.0.0/0"], network: service.networkName, - allow: [{ protocol: "tcp", ports: [std.toString(p) for p in service.fwTcpPorts]}], + allow: [{ protocol: "tcp", ports: [std.toString(p) for p in service.fwTcpPorts] }], target_tags: ["${-}"], - } + }, }, google_compute_instance: error "InstanceBasedService should define some instances.", }, @@ -218,7 +218,7 @@ local apt = import "../cmd/apt.jsonnet"; Mixin+: { zones:: service.zones, }, - + versions:: {}, deployment:: {}, local instances = std.foldl(function(a, b) a + b, [ @@ -228,7 +228,7 @@ local apt = import "../cmd/apt.jsonnet"; service.versions[vname] { name: "${-}-%s-%d" % [vname, i], zone: self.zones[i % std.length(self.zones)], - tags+: [vname, "index-%d" % i] + tags+: [vname, "index-%d" % i], } else error "Undefined version: %s" % vname @@ -307,7 +307,7 @@ local apt = import "../cmd/apt.jsonnet"; }, }, google_dns_record_set: { - } + }, }, outputs+: { "${-}-name_servers": "${google_dns_managed_zone.${-}.name_servers.0}", @@ -330,7 +330,7 @@ local apt = import "../cmd/apt.jsonnet"; ttl: 300, rrdatas: [service.target + "." + service.dnsName], }, - } - } + }, + }, }, } diff --git a/case_studies/micromanage/lib/mmlib/v0.1.0/web/nginx.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.0/web/nginx.jsonnet index 5e36b4826..4c8f526d6 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.0/web/nginx.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.0/web/nginx.jsonnet @@ -50,7 +50,7 @@ local cmd = import "../cmd/cmd.jsonnet"; }, ] else []) + [ "rm /etc/nginx/sites-enabled/default", - ] + ], }, @@ -87,7 +87,7 @@ local cmd = import "../cmd/cmd.jsonnet"; + "--die-on-term --uid www-data --gid www-data\n", filePermissions: "700", }, - ] + ], }, application:: "app", @@ -108,7 +108,7 @@ local cmd = import "../cmd/cmd.jsonnet"; uwsgiModuleContent:: null, - httpContentCmds+: if version.uwsgiModuleContent == null then [ ] else [ + httpContentCmds+: if version.uwsgiModuleContent == null then [] else [ cmd.LiteralFile { content: version.uwsgiModuleContent, to: "/var/www/%s.py" % version.module, @@ -122,14 +122,14 @@ local cmd = import "../cmd/cmd.jsonnet"; to: "/etc/uwsgi/vassals/uwsgi.ini", content: std.manifestIni({ sections: { - uwsgi: version.uwsgiConf - } - }) + uwsgi: version.uwsgiConf, + }, + }), }, cmd.EnsureDir { dir: "/var/log/uwsgi", owner: "www-data" }, "/usr/local/bin/uwsgi --master --emperor /etc/uwsgi/vassals " - + "--daemonize /var/log/uwsgi/emperor.log --pidfile /var/run/uwsgi.pid " - + "--die-on-term --uid www-data --gid www-data\n", + + "--daemonize /var/log/uwsgi/emperor.log --pidfile /var/run/uwsgi.pid " + + "--die-on-term --uid www-data --gid www-data\n", ], }, @@ -148,7 +148,7 @@ local cmd = import "../cmd/cmd.jsonnet"; uwsgi_pass unix:%(socket)s; } } - ||| % {port: version.httpPort, socket: version.uwsgiSocket}, + ||| % { port: version.httpPort, socket: version.uwsgiSocket }, nginxAdditionalCmds+: [ cmd.LiteralFile { @@ -169,7 +169,7 @@ local cmd = import "../cmd/cmd.jsonnet"; |||, Instance+: $.DebianNginxMixin + $.DebianUwsgiFlask + $.NginxUwsgiGlue { httpPort: service.httpPort, - uwsgiModuleContent: service.uwsgiModuleContent + uwsgiModuleContent: service.uwsgiModuleContent, }, }, } diff --git a/case_studies/micromanage/lib/mmlib/v0.1.1/cmd/apt.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.1/cmd/apt.jsonnet index 7c5978783..e9c5a296a 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.1/cmd/apt.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.1/cmd/apt.jsonnet @@ -11,22 +11,22 @@ local pip = import "pip.jsonnet"; aptRepoLines:: {}, // { foo: "..." } will add a foo.list fetched from the given URL. aptRepoUrls:: {}, - aptPackages :: [], + aptPackages:: [], local repoLineCommands = ["echo \"%s\" > %s/%s.list" % [self.aptRepoLines[k], self.aptSourcesListDir, k] for k in std.objectFields(self.aptRepoLines)], local repoUrlCommands = ["curl -o %s/%s.list %s" % [self.aptSourcesListDir, k, self.aptRepoUrls[k]] for k in std.objectFields(self.aptRepoUrls)], - local keyCommands = [ "curl --silent %s | apt-key add -" % [url] for url in self.aptKeyUrls], + local keyCommands = ["curl --silent %s | apt-key add -" % [url] for url in self.aptKeyUrls], local env = "DEBIAN_FRONTEND=noninteractive", local opts = "-o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold", - local updateCommands = [ "apt-get -qq -y update" ], - local installCommands = [ "%s apt-get -qq -y %s install %s" % [env, opts, std.join(" ", self.aptPackages) ]], + local updateCommands = ["apt-get -qq -y update"], + local installCommands = ["%s apt-get -qq -y %s install %s" % [env, opts, std.join(" ", self.aptPackages)]], cmds+: repoLineCommands + repoUrlCommands + keyCommands - + updateCommands + installCommands, + + updateCommands + installCommands, }, PipMixin:: pip.Mixin { diff --git a/case_studies/micromanage/lib/mmlib/v0.1.1/cmd/pip.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.1/cmd/pip.jsonnet index 7360a1562..9df850e9f 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.1/cmd/pip.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.1/cmd/pip.jsonnet @@ -5,7 +5,7 @@ [] else ["pip install " + std.join(" ", self.pipPackages)], - pipPackages :: [], + pipPackages:: [], }, } diff --git a/case_studies/micromanage/lib/mmlib/v0.1.1/db/cassandra.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.1/db/cassandra.jsonnet index e3ef2ae9c..d0e6e949a 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.1/db/cassandra.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.1/db/cassandra.jsonnet @@ -3,7 +3,7 @@ local service_amazon = import "../service/amazon.jsonnet"; // TODO(dcunnin): Separate into 5 mixins: // 1) mixins that sets up cassandra via 1) grabbing JAR from cassandra.org, 2) using Debian -// 2) mixins that sets up infrastructure on 1) GCP 2) AWS +// 2) mixins that sets up infrastructure on 1) GCP 2) AWS // 3) mixin that does the common stuff (different types of nodes, cassandra bootstrap) { @@ -19,7 +19,7 @@ local service_amazon = import "../service/amazon.jsonnet"; client_encryption_options: { enabled: false, keystore: "conf/.keystore", - keystore_password: "cassandra" + keystore_password: "cassandra", }, cluster_name: "Unnamed Cluster", column_index_size_in_kb: 64, @@ -73,15 +73,15 @@ local service_amazon = import "../service/amazon.jsonnet"; seed_provider: [ { class_name: "org.apache.cassandra.locator.SimpleSeedProvider", - parameters: [ { seeds: "127.0.0.1" } ], - } + parameters: [{ seeds: "127.0.0.1" }], + }, ], server_encryption_options: { internode_encryption: "none", keystore: "conf/.keystore", keystore_password: "cassandra", truststore: "conf/.truststore", - truststore_password: "cassandra" + truststore_password: "cassandra", }, snapshot_before_compaction: false, ssl_storage_port: 7001, @@ -95,7 +95,7 @@ local service_amazon = import "../service/amazon.jsonnet"; trickle_fsync: false, trickle_fsync_interval_in_kb: 10240, truncate_request_timeout_in_ms: 60000, - write_request_timeout_in_ms: 2000 + write_request_timeout_in_ms: 2000, }, // A line of bash that will wait for a Cassandra service to be "up". @@ -127,14 +127,14 @@ local service_amazon = import "../service/amazon.jsonnet"; "rm -rfv /var/lib/cassandra/*", // Enable authentication local dest = "/etc/cassandra/cassandra.yaml"; - "echo %s > %s" % [std.escapeStringBash("" + bootstrap_conf), dest], + "echo %s > %s" % [std.escapeStringBash("" + bootstrap_conf), dest], // Start it up again (for some reason 'start' does not do anything...) "/etc/init.d/cassandra restart", // Wait for it to be ready wait_for_cqlsh("cassandra", "cassandra", "localhost"), // Set root password local cql = "ALTER USER cassandra WITH PASSWORD '%s';" % node.rootPassword; - "echo %s | cqlsh -u cassandra -p cassandra" % std.escapeStringBash(cql), + "echo %s | cqlsh -u cassandra -p cassandra" % std.escapeStringBash(cql), ], }, @@ -155,13 +155,13 @@ local service_amazon = import "../service/amazon.jsonnet"; // Set up system_auth replication level "echo %s | cqlsh -u cassandra -p %s localhost" - % [std.escapeStringBash("ALTER KEYSPACE system_auth WITH REPLICATION = %s;" - % self.initAuthReplication), - self.rootPassword], + % [std.escapeStringBash("ALTER KEYSPACE system_auth WITH REPLICATION = %s;" + % self.initAuthReplication), + self.rootPassword], // Drop in the correct configuration. "echo %s > %s" - % [std.escapeStringBash("" + self.conf), "/etc/cassandra/cassandra.yaml"], + % [std.escapeStringBash("" + self.conf), "/etc/cassandra/cassandra.yaml"], // Restart with new configuration. "/etc/init.d/cassandra restart", @@ -171,8 +171,8 @@ local service_amazon = import "../service/amazon.jsonnet"; // Set up users, empty tables, etc. "echo %s | cqlsh -u cassandra -p %s $HOSTNAME" - % [std.escapeStringBash(std.lines(self.initCql)), self.rootPassword], - ] + % [std.escapeStringBash(std.lines(self.initCql)), self.rootPassword], + ], }, // Simply restarts with the given config. @@ -189,12 +189,12 @@ local service_amazon = import "../service/amazon.jsonnet"; // Drop in the correct configuration. "echo %s > %s" - % [std.escapeStringBash("" + self.conf), "/etc/cassandra/cassandra.yaml"], + % [std.escapeStringBash("" + self.conf), "/etc/cassandra/cassandra.yaml"], // Start it up again. "/etc/init.d/cassandra restart", - ] + ], }, GcpDebianCassandra: service_google.Service { @@ -320,8 +320,8 @@ local service_amazon = import "../service/amazon.jsonnet"; obj: "org.apache.cassandra.db:type=StorageProxy", attr: ["RecentReadLatencyMicros", "RecentWriteLatencyMicros", "RecentRangeLatencyMicros", "HintsInProgress"], - } - ] + }, + ], }, enableLogging: self.supportsLogging, @@ -347,7 +347,7 @@ local service_amazon = import "../service/amazon.jsonnet"; { disk: "${google_compute_disk.${-}-%s.name}" % n, auto_delete: false, - } + }, ], } for n in std.objectFields(service.nodes) @@ -360,11 +360,11 @@ local service_amazon = import "../service/amazon.jsonnet"; allow: [ { protocol: "tcp", - ports: [std.toString(p) for p in service.tcpFirewallPorts] - } + ports: [std.toString(p) for p in service.tcpFirewallPorts], + }, ], target_tags: [service.clusterName], - } + }, }, }, }, diff --git a/case_studies/micromanage/lib/mmlib/v0.1.1/service/amazon.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.1/service/amazon.jsonnet index d3ca012ee..b935a8fb1 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.1/service/amazon.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.1/service/amazon.jsonnet @@ -50,7 +50,7 @@ local amis_debian = import "../amis/debian.jsonnet"; Service:: base.Service { infrastructure+: if self.dnsZone == null then { } else { -/* + /* local instances = if std.objectHas(self, "aws_instance") then self.google_compute_instance else { }, local addresses = { }, //if std.objectHas(self, "google_compute_address") then self.google_compute_address else { }, local DnsRecord = { @@ -70,7 +70,7 @@ local amis_debian = import "../amis/debian.jsonnet"; rrdatas: ["${google_compute_instance." + name + ".network_interface.0.access_config.0.nat_ip}"], } for name in std.objectFields(instances) }, -*/ + */ }, dnsZone:: null, dnsZoneName:: error "Must set dnsZoneName if dnsZone is set.", @@ -156,10 +156,10 @@ local amis_debian = import "../amis/debian.jsonnet"; cidr_blocks: ["0.0.0.0/0"], }, ingress: [IngressRule(p, "tcp") for p in service.fwTcpPorts] - + [IngressRule(p, "udp") for p in service.fwUdpPorts], + + [IngressRule(p, "udp") for p in service.fwUdpPorts], [if service.networkName != null then "vpc_id"]: $.Network.refId(service.networkName), - } + }, }, aws_instance: error "InstanceBasedService should define some instances.", }, @@ -182,7 +182,8 @@ local amis_debian = import "../amis/debian.jsonnet"; }, versions:: {}, deployment:: {}, - local instances = std.foldl(function(a, b) a + b, [ + local merge(objs) = std.fold(function(a, b) a + b, objs, {}), + local instances = merge([ { ["${-}-%s-%d" % [vname, i]]: if std.objectHas(service.versions, vname) then @@ -191,14 +192,14 @@ local amis_debian = import "../amis/debian.jsonnet"; tags+: { version: vname, index: i, - } + }, } else error "Undefined version: %s" % vname for i in std.set(service.deployment[vname].deployed) } for vname in std.objectFields(service.deployment) - ], {}), + ]), local attached_instances = std.join([], [ local attached = std.set(service.deployment[vname].attached); local deployed = std.set(service.deployment[vname].deployed); @@ -250,10 +251,10 @@ local amis_debian = import "../amis/debian.jsonnet"; cidr_blocks: ["0.0.0.0/0"], }, ingress: [IngressRule(p, "tcp") for p in service.lbTcpPorts] - + [IngressRule(p, "udp") for p in service.lbUdpPorts], + + [IngressRule(p, "udp") for p in service.lbUdpPorts], //[if service.networkName != null then "vpc_id"]: $.Network.refId(service.networkName), - } + }, }, }, }, @@ -287,7 +288,7 @@ local amis_debian = import "../amis/debian.jsonnet"; }, }, aws_dns_record: { - } + }, }, outputs+: { "${-}-name_servers": "${google_dns_managed_zone.${-}.name_servers.0}", @@ -295,7 +296,7 @@ local amis_debian = import "../amis/debian.jsonnet"; }, -/* + /* DnsRecordWww:: self.Service { local service = self, dnsName:: service.zone.dnsName, @@ -313,7 +314,7 @@ local amis_debian = import "../amis/debian.jsonnet"; }, } } - }, -*/ + }, + */ } diff --git a/case_studies/micromanage/lib/mmlib/v0.1.1/service/google.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.1/service/google.jsonnet index c292b3aa6..768d93388 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.1/service/google.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.1/service/google.jsonnet @@ -36,17 +36,17 @@ local apt = import "../cmd/apt.jsonnet"; network: instance.networkName, access_config: { }, - }, + }, service_account: [ { scopes: ["https://www.googleapis.com/auth/" + s for s in instance.scopes], - } - ], + }, + ], disk: [ - { + { image: instance.StandardRootImage, - } - ], + }, + ], supportsLogging:: true, supportsMonitoring:: true, @@ -58,19 +58,19 @@ local apt = import "../cmd/apt.jsonnet"; jmxPort:: 9012, jmxHotspotConfig:: { host: instance.jmxHost, - port : instance.jmxPort, - numQueryThreads : 2, + port: instance.jmxPort, + numQueryThreads: 2, SdQuery:: { outputWriters: [ - { - "@class" : "com.googlecode.jmxtrans.model.output.StackdriverWriter", - settings: { - token: "STACKDRIVER_API_KEY", - detectInstance: "GCE", - url: "https://jmx-gateway.google.stackdriver.com/v1/custom" - } - } - ], + { + "@class": "com.googlecode.jmxtrans.model.output.StackdriverWriter", + settings: { + token: "STACKDRIVER_API_KEY", + detectInstance: "GCE", + url: "https://jmx-gateway.google.stackdriver.com/v1/custom", + }, + }, + ], }, queries: [ self.SdQuery { @@ -89,8 +89,8 @@ local apt = import "../cmd/apt.jsonnet"; attr: ["Uptime"], }, self.SdQuery { - resultAlias : "jvm.localhost.os", - obj : "java.lang:type=OperatingSystem", + resultAlias: "jvm.localhost.os", + obj: "java.lang:type=OperatingSystem", attr: [ "CommittedVirtualMemorySize", "FreePhysicalMemorySize", @@ -98,20 +98,20 @@ local apt = import "../cmd/apt.jsonnet"; "OpenFileDescriptorCount", "ProcessCpuTime", "SystemLoadAverage", - ] + ], }, self.SdQuery { resultAlias: "jvm.localhost.gc", obj: "java.lang:type=GarbageCollector,name=*", attr: ["CollectionCount", "CollectionTime"], - } + }, ], }, jmxLocalhostConfig:: instance.jmxHotspotConfig, jmxConfig:: { servers: [ instance.jmxLocalhostConfig, - ] + ], }, MonitoringLoggingImageMixin:: { @@ -151,8 +151,8 @@ local apt = import "../cmd/apt.jsonnet"; local service = self, infrastructure+: if self.dnsZone == null then { } else { - local instances = if std.objectHas(self, "google_compute_instance") then self.google_compute_instance else { }, - local addresses = if std.objectHas(self, "google_compute_address") then self.google_compute_address else { }, + local instances = if std.objectHas(self, "google_compute_instance") then self.google_compute_instance else {}, + local addresses = if std.objectHas(self, "google_compute_address") then self.google_compute_address else {}, local DnsRecord = { managed_zone: service.dnsZone.refName(service.dnsZoneName), type: "A", @@ -185,8 +185,8 @@ local apt = import "../cmd/apt.jsonnet"; "${-}": { name: "${-}", ipv4_range: service.ipv4Range, - } - } + }, + }, }, }, @@ -207,16 +207,16 @@ local apt = import "../cmd/apt.jsonnet"; infrastructure+: { google_compute_address: { - "${-}": { name: "${-}" } + "${-}": { name: "${-}" }, }, google_compute_firewall: { "${-}": { name: "${-}", source_ranges: ["0.0.0.0/0"], network: service.networkName, - allow: [{ protocol: "tcp", ports: [std.toString(p) for p in service.fwTcpPorts]}], + allow: [{ protocol: "tcp", ports: [std.toString(p) for p in service.fwTcpPorts] }], target_tags: ["${-}"], - } + }, }, google_compute_instance: error "InstanceBasedService should define some instances.", }, @@ -233,7 +233,7 @@ local apt = import "../cmd/apt.jsonnet"; Mixin+: { zones:: service.zones, }, - + versions:: {}, deployment:: {}, local instances = std.foldl(function(a, b) a + b, [ @@ -243,7 +243,7 @@ local apt = import "../cmd/apt.jsonnet"; service.versions[vname] { name: "${-}-%s-%d" % [vname, i], zone: self.zones[i % std.length(self.zones)], - tags+: [vname, "index-%d" % i] + tags+: [vname, "index-%d" % i], } else error "Undefined version: %s" % vname @@ -251,12 +251,13 @@ local apt = import "../cmd/apt.jsonnet"; } for vname in std.objectFields(service.deployment) ], {}), - local attached_instances = std.join([], [ - local attached = std.set(service.deployment[vname].attached); - local deployed = std.set(service.deployment[vname].deployed); - ["${-}-%s-%d" % [vname, i] for i in std.setInter(attached, deployed)] - for vname in std.objectFields(service.deployment) - ]), + local attached_instances = std.join( + [], [ + local attached = std.set(service.deployment[vname].attached); + local deployed = std.set(service.deployment[vname].deployed); + ["${-}-%s-%d" % [vname, i] for i in std.setInter(attached, deployed)] + for vname in std.objectFields(service.deployment) + ]), infrastructure+: { google_compute_http_health_check: { @@ -322,7 +323,7 @@ local apt = import "../cmd/apt.jsonnet"; }, }, google_dns_record_set: { - } + }, }, outputs+: { "${-}-name_servers": "${google_dns_managed_zone.${-}.name_servers.0}", @@ -345,7 +346,7 @@ local apt = import "../cmd/apt.jsonnet"; ttl: 300, rrdatas: [service.target + "." + service.dnsName], }, - } - } + }, + }, }, } diff --git a/case_studies/micromanage/lib/mmlib/v0.1.1/web/nginx.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.1/web/nginx.jsonnet index ea8ce61d7..aba1e45c9 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.1/web/nginx.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.1/web/nginx.jsonnet @@ -51,7 +51,7 @@ local cmd = import "../cmd/cmd.jsonnet"; }, ] else []) + [ "rm /etc/nginx/sites-enabled/default", - ] + ], }, diff --git a/case_studies/micromanage/lib/mmlib/v0.1.1/web/solutions.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.1/web/solutions.jsonnet index a9e393136..03b8acb97 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.1/web/solutions.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.1/web/solutions.jsonnet @@ -14,7 +14,7 @@ local uwsgi_flask = import "uwsgi_flask.jsonnet"; |||, Instance+: nginx.DebianNginxMixin + uwsgi_flask.DebianUwsgiFlask + uwsgi_flask.NginxUwsgiGlue { httpPort: service.httpPort, - uwsgiModuleContent: service.uwsgiModuleContent + uwsgiModuleContent: service.uwsgiModuleContent, }, }, } diff --git a/case_studies/micromanage/lib/mmlib/v0.1.1/web/uwsgi_flask.jsonnet b/case_studies/micromanage/lib/mmlib/v0.1.1/web/uwsgi_flask.jsonnet index c73a37e5c..2dbc4a84c 100644 --- a/case_studies/micromanage/lib/mmlib/v0.1.1/web/uwsgi_flask.jsonnet +++ b/case_studies/micromanage/lib/mmlib/v0.1.1/web/uwsgi_flask.jsonnet @@ -17,7 +17,7 @@ local cmd = import "../cmd/cmd.jsonnet"; + "--die-on-term --uid www-data --gid www-data\n", filePermissions: "700", }, - ] + ], }, application:: "app", @@ -38,7 +38,7 @@ local cmd = import "../cmd/cmd.jsonnet"; uwsgiModuleContent:: null, - httpContentCmds+: if version.uwsgiModuleContent == null then [ ] else [ + httpContentCmds+: if version.uwsgiModuleContent == null then [] else [ cmd.LiteralFile { content: version.uwsgiModuleContent, to: "/var/www/%s.py" % version.module, @@ -52,14 +52,14 @@ local cmd = import "../cmd/cmd.jsonnet"; to: "/etc/uwsgi/vassals/uwsgi.ini", content: std.manifestIni({ sections: { - uwsgi: version.uwsgiConf - } - }) + uwsgi: version.uwsgiConf, + }, + }), }, cmd.EnsureDir { dir: "/var/log/uwsgi", owner: "www-data" }, "/usr/local/bin/uwsgi --master --emperor /etc/uwsgi/vassals " - + "--daemonize /var/log/uwsgi/emperor.log --pidfile /var/run/uwsgi.pid " - + "--die-on-term --uid www-data --gid www-data\n", + + "--daemonize /var/log/uwsgi/emperor.log --pidfile /var/run/uwsgi.pid " + + "--die-on-term --uid www-data --gid www-data\n", ], }, @@ -79,7 +79,7 @@ local cmd = import "../cmd/cmd.jsonnet"; uwsgi_pass unix:%(socket)s; } } - ||| % {port: version.httpPort, socket: version.uwsgiSocket}, + ||| % { port: version.httpPort, socket: version.uwsgiSocket }, nginxAdditionalCmds+: [ cmd.LiteralFile { diff --git a/case_studies/micromanage/tests/amazon/test_single_instance.jsonnet b/case_studies/micromanage/tests/amazon/test_single_instance.jsonnet index 3e8b0c5ae..7a88867a1 100644 --- a/case_studies/micromanage/tests/amazon/test_single_instance.jsonnet +++ b/case_studies/micromanage/tests/amazon/test_single_instance.jsonnet @@ -30,8 +30,8 @@ local amis_debian = import "mmlib/v0.1.1/amis/debian.jsonnet"; ami: if service.ami != null then service.ami else service.amiMap[region_from_zone(service.zone)], associate_public_ip_address: service.externalIp, cmds: service.cmds, - } - } + }, + }, }, outputs: { "${-}-address": "${aws_instance.${-}.public_ip}", diff --git a/case_studies/micromanage/tests/google/test_nested.jsonnet b/case_studies/micromanage/tests/google/test_nested.jsonnet index c8bb8fc3c..c238fcf39 100644 --- a/case_studies/micromanage/tests/google/test_nested.jsonnet +++ b/case_studies/micromanage/tests/google/test_nested.jsonnet @@ -22,19 +22,19 @@ }], network_interface: { network: "default", - access_config: - if service.externalIp then [{}] else [] + access_config: + if service.externalIp then [{}] else [], }, metadata: { }, cmds: [], bootCmds: [], - } - } + }, + }, }, outputs: { [if service.externalIp then "address"]: - "${google_compute_instance.${-}.network_interface.access_config.0.nat_ip}" + "${google_compute_instance.${-}.network_interface.access_config.0.nat_ip}", }, }, @@ -54,5 +54,5 @@ zone: "us-central1-f", externalIp: true, }, - } + }, } diff --git a/case_studies/micromanage/tests/google/test_single_instance.jsonnet b/case_studies/micromanage/tests/google/test_single_instance.jsonnet index 06797ce05..7f030779b 100644 --- a/case_studies/micromanage/tests/google/test_single_instance.jsonnet +++ b/case_studies/micromanage/tests/google/test_single_instance.jsonnet @@ -25,19 +25,19 @@ local cmd = import "mmlib/v0.1.0/cmd/cmd.jsonnet"; }], network_interface: { network: "default", - access_config: - if service.externalIp then [{}] else [] + access_config: + if service.externalIp then [{}] else [], }, metadata: { }, cmds: service.cmds, bootCmds: [], - } - } + }, + }, }, outputs: { [if service.externalIp then "${-}-address"]: - "${google_compute_instance.${-}.network_interface.access_config.0.nat_ip}" + "${google_compute_instance.${-}.network_interface.access_config.0.nat_ip}", }, }, diff --git a/cmd/jsonnet.cpp b/cmd/jsonnet.cpp index 8f848b96c..a8d234429 100644 --- a/cmd/jsonnet.cpp +++ b/cmd/jsonnet.cpp @@ -72,10 +72,10 @@ void version(std::ostream &o) void usage(std::ostream &o) { version(o); - o << "Usage:\n"; - o << "jsonnet {