Skip to content

Commit

Permalink
Merge pull request #37 from invenia/1.2.1-unbreak
Browse files Browse the repository at this point in the history
1.2.1 unbreak
  • Loading branch information
rofinn authored Nov 20, 2019
2 parents 64f3937 + e8e02c0 commit d2118a4
Show file tree
Hide file tree
Showing 20 changed files with 393 additions and 84 deletions.
9 changes: 7 additions & 2 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name = "JLSO"
uuid = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11"
license = "MIT"
authors = ["Invenia Technical Computing Corperation"]
version = "1.2.1"
version = "1.2.2"

[deps]
BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
Expand All @@ -21,9 +21,14 @@ julia = "0.7, 1.0"
[extras]
AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"

[targets]
test = ["AxisArrays", "DataFrames", "Distributions", "Test", "TimeZones"]
test = ["AxisArrays", "DataFrames", "Dates", "Distributed", "Distributions", "InteractiveUtils", "Random", "Suppressor", "Test", "TimeZones"]
14 changes: 8 additions & 6 deletions src/JLSO.jl
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@ Dict(
"format" => :bson, # Could also be :julia_serialize
"compression" => :gzip_fastest, # could also be: :none, :gzip_smallest, or :gzip
"image" => "xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/myrepository:latest"
"pkgs" => Dict(
"AxisArrays" => v"0.2.1",
...
)
"project" => Dict{String, Any}(...),
"manifest" => Dict{String, Any}(...),
),
"objects" => Dict(
"var1" => [0x35, 0x10, 0x01, 0x04, 0x44],
Expand All @@ -44,15 +42,19 @@ using Memento
using Pkg: Pkg
using Pkg.Types: semver_spec

# We need to import these cause of a deprecation on object index via strings.
import Base: getindex, setindex!

export JLSOFile

const READABLE_VERSIONS = semver_spec("1, 2")
const WRITEABLE_VERSIONS = semver_spec("2")
const READABLE_VERSIONS = semver_spec("1, 2, 3")
const WRITEABLE_VERSIONS = semver_spec("3")

const LOGGER = getlogger(@__MODULE__)
__init__() = Memento.register(LOGGER)

include("JLSOFile.jl")
include("deprecated.jl")
include("file_io.jl")
include("metadata.jl")
include("serialization.jl")
Expand Down
63 changes: 50 additions & 13 deletions src/JLSOFile.jl
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ struct JLSOFile
format::Symbol
compression::Symbol
image::String
pkgs::Dict{String, VersionNumber}
objects::Dict{String, Vector{UInt8}}
project::Dict{String, Any}
manifest::Dict{String, Any}
objects::Dict{Symbol, Vector{UInt8}}
end

"""
Expand All @@ -31,8 +32,8 @@ Stores the information needed to write a .jlso file.
Due to the time taken for disk IO, :none is not normally as fast as using some compression.
"""
function JLSOFile(
data::Dict{String, <:Any};
version=v"2.0.0",
data::Dict{Symbol, <:Any};
version=v"3",
julia=VERSION,
format=:julia_serialize,
compression=:gzip,
Expand All @@ -45,8 +46,17 @@ function JLSOFile(
end

_versioncheck(version, WRITEABLE_VERSIONS)
objects = Dict{String, Vector{UInt8}}()
jlso = JLSOFile(version, julia, format, compression, image, _pkgs(), objects)
project_toml, manifest_toml = _env()
jlso = JLSOFile(
version,
julia,
format,
compression,
image,
Pkg.TOML.parse(project_toml),
Pkg.TOML.parse(manifest_toml),
Dict{Symbol, Vector{UInt8}}(),
)

for (key, val) in data
jlso[key] = val
Expand All @@ -55,15 +65,33 @@ function JLSOFile(
return jlso
end

JLSOFile(data; kwargs...) = JLSOFile(Dict("data" => data); kwargs...)
JLSOFile(data::Pair...; kwargs...) = JLSOFile(Dict(data...); kwargs...)
function JLSOFile(;
version=v"3",
julia=VERSION,
format=:julia_serialize,
compression=:gzip,
image=_image(),
kwargs...
)
return JLSOFile(
Dict(kwargs);
version=version,
julia=julia,
format=format,
compression=compression,
image=image,
)
end


JLSOFile(data::Pair...; kwargs...) = JLSOFile(Dict(data); kwargs...)

function Base.show(io::IO, jlso::JLSOFile)
variables = join(names(jlso), ", ")
kwargs = join(
[
"version=v\"$(jlso.version)\"",
"julia=v\"$(jlso.julia)\"",
"version=\"$(jlso.version)\"",
"julia=\"$(jlso.julia)\"",
"format=:$(jlso.format)",
"compression=:$(jlso.compression)",
"image=\"$(jlso.image)\"",
Expand All @@ -79,13 +107,22 @@ function Base.:(==)(a::JLSOFile, b::JLSOFile)
a.version == b.version &&
a.julia == b.julia &&
a.image == b.image &&
a.pkgs == b.pkgs &&
a.manifest == b.manifest &&
a.format == b.format &&
a.compression == b.compression &&
a.objects == b.objects
)
end

Base.names(jlso::JLSOFile) = collect(keys(jlso.objects))

# TODO: Include a more detail summary method for displaying version information.
function Base.getproperty(jlso::JLSOFile, attr::Symbol)
if attr === :pkgs
@warn "pkgs property is deprecated, use .manifest instead"
return Dict(
name => VersionNumber(get(first(spec), "version", "0.0.0"))
for (name, spec) in jlso.manifest
)
else
return getfield(jlso, attr)
end
end
123 changes: 123 additions & 0 deletions src/deprecated.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
@deprecate(
JLSOFile(
version::VersionNumber,
julia::VersionNumber,
format::Symbol,
compression::Symbol,
image::String,
pkgs::Dict{String, VersionNumber},
objects::Dict{String, Vector{UInt8}},
),
JLSOFile(
version,
julia,
format,
compression,
image,
Pkg.TOML.parse.(_upgrade_env(pkgs))...,
Dict{Symbol, Vector{UInt8}}(Symbol(k) => v for (k, v) in objects),
)
)

@deprecate JLSOFile(data; kwargs...) JLSOFile(:data => data; kwargs...)
@deprecate(
JLSOFile(data::Dict{String, <:Any}; kwargs...),
JLSOFile(Dict(Symbol(k) => v for (k, v) in data); kwargs...)
)

function upgrade_jlso(raw_dict::AbstractDict)
result = copy(raw_dict)
version = version_number(result["metadata"]["version"])

while version < v"3"
result = upgrade_jlso(result, Val(Int(version.major)))
version = version_number(result["metadata"]["version"])
end

return result
end

@deprecate getindex(jlso::JLSOFile, name::String) getindex(jlso, Symbol(name))
@deprecate setindex!(jlso::JLSOFile, value, name::String) setindex!(jlso, value, Symbol(name))

# v1 and v2 stored version numbers but v3 stores strings to be bson parser agnostic
version_number(x::VersionNumber) = x
version_number(x::String) = VersionNumber(x)

# Metadata changes to upgrade file from v1 to v2
function upgrade_jlso(raw_dict::AbstractDict, ::Val{1})
info(LOGGER, "Upgrading JLSO format from v1 to v2")
metadata = copy(raw_dict["metadata"])
version = version_number(metadata["version"])
@assert version semver_spec("1")

if metadata["format"] == :serialize
metadata["format"] = :julia_serialize
end

metadata["compression"] = :none
metadata["version"] = v"2"

return Dict("metadata" => metadata, "objects" => get(raw_dict, "objects", Dict()))
end

# Metadata changes to upgrade from v2 to v3
function upgrade_jlso(raw_dict::AbstractDict, ::Val{2})
info(LOGGER, "Upgrading JLSO format from v2 to v3")
metadata = copy(raw_dict["metadata"])
version = version_number(metadata["version"])
@assert version semver_spec("2")

# JLSO v3 expects to be loading a compressed manifest
compression = get(metadata, "compression", :none)
project, manifest = _upgrade_env(get(metadata, "pkgs", nothing))

return Dict{String, Dict}(
"metadata" => Dict{String, Union{String, Vector{UInt8}}}(
"version" => "3",
"julia" => string(get(metadata, "julia", VERSION)),
"format" => string(get(metadata, "format", :julia_serialize)),
"compression" => string(compression),
"image" => get(metadata, "image", ""),
"project" => project,
"manifest" => read(compress(compression, IOBuffer(manifest))),
),
"objects" => Dict{String, Vector{UInt8}}(
k => v for (k, v) in get(raw_dict, "objects", Dict())
),
)
end

# Used to convert the old "PkgName" => VersionNumber metadata to a
# Project.toml and Manifest.toml file.
function _upgrade_env(pkgs::Dict)
src_env = Base.active_project()

try
mktempdir() do tmp
Pkg.activate(tmp)

# We construct an array of PackageSpecs to avoid ordering problems with
# adding each package individually
try
Pkg.add([
Pkg.PackageSpec(; name=key, version=value) for (key, value) in pkgs
])
catch e
# Warn about failure and fallback to simply trying to install the pacakges
# without version constraints.
warn(LOGGER) do
"Failed to construct an environment with the provide package version " *
"($pkgs): $e.\n Falling back to simply adding the packages."
end
Pkg.add([Pkg.PackageSpec(; name=key) for (key, value) in pkgs])
end

return _env()
end
finally
Pkg.activate(src_env)
end
end

_upgrade_env(::Nothing) = ("", "")
Loading

2 comments on commit d2118a4

@rofinn
Copy link
Member Author

@rofinn rofinn commented on d2118a4 Nov 20, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/5663

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if Julia TagBot is installed, or can be done manually through the github interface, or via:

git tag -a v1.2.2 -m "<description of version>" d2118a4fca043cfad1ead20780c1ebd0dbdda79c
git push origin v1.2.2

Please sign in to comment.