From 88727986ffa91662593958023be8ac3ccef2cab0 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Sun, 7 Apr 2024 21:10:55 +0300 Subject: [PATCH 1/8] [onnx utils] update onnx utils packages --- onnx_utils/function.yaml | 42 +++++++++++++++++------------------ onnx_utils/item.yaml | 16 ++++++------- onnx_utils/requirements.txt | 12 +++++----- onnx_utils/test_onnx_utils.py | 4 ++-- 4 files changed, 36 insertions(+), 38 deletions(-) diff --git a/onnx_utils/function.yaml b/onnx_utils/function.yaml index 7a0054c4d..88f810fb4 100644 --- a/onnx_utils/function.yaml +++ b/onnx_utils/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: onnx-utils tag: '' - hash: 0c4a6491b976d5220d3ebfb83a3905dd47e86be2 + hash: fd6cd909ef6e055c348b44a0313e190513cd755b project: '' labels: author: guyl @@ -16,16 +16,16 @@ spec: functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgQ2FsbGFibGUsIERpY3QsIExpc3QsIFR1cGxlCgppbXBvcnQgbWxydW4KCgpjbGFzcyBfVG9PTk5YQ29udmVyc2lvbnM6CiAgICAiIiIKICAgIEFuIE9OTlggY29udmVyc2lvbiBmdW5jdGlvbnMgbGlicmFyeSBjbGFzcy4KICAgICIiIgoKICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiB0Zl9rZXJhc190b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50XSwgc3RyXV0gPSBOb25lLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBDb252ZXJ0IGEgVEYuS2VyYXMgbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICBBbiBpbml0aWFsaXplZCBURktlcmFzTW9kZWxIYW5kbGVyIHdpdGggYSBsb2FkZWQgbW9kZWwgdG8gY29udmVydCB0byBPTk5YLgogICAgICAgIDpwYXJhbSBvbm54X21vZGVsX25hbWU6IFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICBXaGV0aGVyIG9yIG5vdCB0byBvcHRpbWl6ZSB0aGUgT05OWCBtb2RlbCB1c2luZyAnb25ueG9wdGltaXplcicgYmVmb3JlIHNhdmluZyB0aGUgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgRGVmYXVsdGVkIHRvIFRydWUuCiAgICAgICAgOnBhcmFtIGlucHV0X3NpZ25hdHVyZTogQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEgbGlzdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdoZXJlIGVhY2ggZWxlbWVudCBpcyBhbiBpbnB1dCBsYXllciB0dXBsZS4gQW4gaW5wdXQgbGF5ZXIgdHVwbGUgaXMgYSB0dXBsZSBvZjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMF0gPSBMYXllcidzIHNoYXBlLCBhIHR1cGxlIG9mIGludGVnZXJzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBOb25lLCB0aGUgaW5wdXQgc2lnbmF0dXJlIHdpbGwgYmUgdHJpZWQgdG8gYmUgcmVhZCBmcm9tIHRoZSBtb2RlbCBhcnRpZmFjdC4gRGVmYXVsdGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG8gTm9uZS4KICAgICAgICAiIiIKICAgICAgICAjIEltcG9ydCB0aGUgZnJhbWV3b3JrIGFuZCBoYW5kbGVyOgogICAgICAgIGltcG9ydCB0ZW5zb3JmbG93IGFzIHRmCiAgICAgICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLnRmX2tlcmFzIGltcG9ydCBURktlcmFzVXRpbHMKCiAgICAgICAgIyBDaGVjayB0aGUgZ2l2ZW4gJ2lucHV0X3NpZ25hdHVyZScgcGFyYW1ldGVyOgogICAgICAgIGlmIGlucHV0X3NpZ25hdHVyZSBpcyBOb25lOgogICAgICAgICAgICAjIFJlYWQgdGhlIGlucHV0cyBmcm9tIHRoZSBtb2RlbDoKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgbW9kZWxfaGFuZGxlci5yZWFkX2lucHV0c19mcm9tX21vZGVsKCkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlcnJvcjoKICAgICAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1blJ1bnRpbWVFcnJvcigKICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSBwcm92aWRlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXIuIFRoZSBmdW5jdGlvbiB0cmllZCByZWFkaW5nIHRoZSBpbnB1dCBsYXllcnMgIgogICAgICAgICAgICAgICAgICAgIGYiaW5mb3JtYXRpb24gYXV0b21hdGljYWxseSBidXQgZmFpbGVkIHdpdGggdGhlIGZvbGxvd2luZyBlcnJvcjoge2Vycm9yfSIKICAgICAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICAjIFBhcnNlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXI6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IFsKICAgICAgICAgICAgICAgIHRmLlRlbnNvclNwZWMoCiAgICAgICAgICAgICAgICAgICAgc2hhcGU9c2hhcGUsCiAgICAgICAgICAgICAgICAgICAgZHR5cGU9VEZLZXJhc1V0aWxzLmNvbnZlcnRfdmFsdWVfdHlwZV90b190Zl9kdHlwZSgKICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICBdCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICkKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgcHl0b3JjaF90b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50LCAuLi5dLCBzdHJdXSA9IE5vbmUsCiAgICAgICAgaW5wdXRfbGF5ZXJzX25hbWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIG91dHB1dF9sYXllcnNfbmFtZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgZHluYW1pY19heGVzOiBEaWN0W3N0ciwgRGljdFtpbnQsIHN0cl1dID0gTm9uZSwKICAgICAgICBpc19iYXRjaGVkOiBib29sID0gVHJ1ZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBhIFB5VG9yY2ggbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICAgICAgQW4gaW5pdGlhbGl6ZWQgUHlUb3JjaE1vZGVsSGFuZGxlciB3aXRoIGEgbG9hZGVkIG1vZGVsIHRvIGNvbnZlcnQgdG8gT05OWC4KICAgICAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgICAgVGhlIG5hbWUgdG8gdXNlIHRvIGxvZyB0aGUgY29udmVydGVkIE9OTlggbW9kZWwuIElmIG5vdCBnaXZlbiwgdGhlIGdpdmVuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtb2RlbF9uYW1lYCB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgICAgV2hldGhlciBvciBub3QgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgICAgICA6cGFyYW0gaW5wdXRfc2lnbmF0dXJlOiAgICAgQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdCB3aGVyZSBlYWNoIGVsZW1lbnQgaXMgYW4gaW5wdXQgbGF5ZXIgdHVwbGUuIEFuIGlucHV0IGxheWVyIHR1cGxlIGlzIGEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFswXSA9IExheWVyJ3Mgc2hhcGUsIGEgdHVwbGUgb2YgaW50ZWdlcnMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgdGhlIGlucHV0IHNpZ25hdHVyZSB3aWxsIGJlIHRyaWVkIHRvIGJlIHJlYWQgZnJvbSB0aGUgbW9kZWwgYXJ0aWZhY3QuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpbnB1dF9sYXllcnNfbmFtZXM6ICBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgaW5wdXQgbm9kZXMgb2YgdGhlIGdyYXBoIGluIG9yZGVyLiBBbGwgb2YgdGhlIG90aGVyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlcnMgKGlubmVyIGxheWVycykgY2FuIGJlIHNldCBhcyB3ZWxsIGJ5IHBhc3NpbmcgYWRkaXRpb25hbCBuYW1lcyBpbiB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdC4gVGhlIG9yZGVyIGlzIGJ5IHRoZSBvcmRlciBvZiB0aGUgcGFyYW1ldGVycyBpbiB0aGUgbW9kZWwuIElmIE5vbmUsIHRoZSBpbnB1dHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBpbnB1dHMuIElmIGl0cyBhbHNvIE5vbmUsIGl0IGlzIGRlZmF1bHRlZCB0bzoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlucHV0XzAiLCAiaW5wdXRfMSIsIC4uLgogICAgICAgIDpwYXJhbSBvdXRwdXRfbGF5ZXJzX25hbWVzOiBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgb3V0cHV0IG5vZGVzIG9mIHRoZSBncmFwaCBpbiBvcmRlci4gSWYgTm9uZSwgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG91dHB1dHMgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBvdXRwdXRzLiBJZiBpdHMgYWxzbyBOb25lLCBpdCBpcyBkZWZhdWx0ZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG86ICJvdXRwdXRfMCIgKGZvciBtdWx0aXBsZSBvdXRwdXRzLCB0aGlzIHBhcmFtZXRlciBtdXN0IGJlIHByb3ZpZGVkKS4KICAgICAgICA6cGFyYW0gZHluYW1pY19heGVzOiAgICAgICAgSWYgcGFydCBvZiB0aGUgaW5wdXQgLyBvdXRwdXQgc2hhcGUgaXMgZHluYW1pYywgbGlrZSAoYmF0Y2hfc2l6ZSwgMywgMzIsIDMyKSB5b3UgY2FuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNwZWNpZnkgaXQgYnkgZ2l2aW5nIGEgZHluYW1pYyBheGlzIHRvIHRoZSBpbnB1dCAvIG91dHB1dCBsYXllciBieSBpdHMgbmFtZSBhcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb2xsb3dzOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiaW5wdXQgbGF5ZXIgbmFtZSI6IHswOiAiYmF0Y2hfc2l6ZSJ9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm91dHB1dCBsYXllciBuYW1lIjogezA6ICJiYXRjaF9zaXplIn0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgcHJvdmlkZWQsIHRoZSAnaXNfYmF0Y2hlZCcgZmxhZyB3aWxsIGJlIGlnbm9yZWQuIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpc19iYXRjaGVkOiAgICAgICAgICBXaGV0aGVyIHRvIGluY2x1ZGUgYSBiYXRjaCBzaXplIGFzIHRoZSBmaXJzdCBheGlzIGluIGV2ZXJ5IGlucHV0IGFuZCBvdXRwdXQgbGF5ZXIuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBUcnVlLiBXaWxsIGJlIGlnbm9yZWQgaWYgJ2R5bmFtaWNfYXhlcycgaXMgcHJvdmlkZWQuCiAgICAgICAgIiIiCiAgICAgICAgIyBJbXBvcnQgdGhlIGZyYW1ld29yayBhbmQgaGFuZGxlcjoKICAgICAgICBpbXBvcnQgdG9yY2gKICAgICAgICBmcm9tIG1scnVuLmZyYW1ld29ya3MucHl0b3JjaCBpbXBvcnQgUHlUb3JjaFV0aWxzCgogICAgICAgICMgUGFyc2UgdGhlICdpbnB1dF9zaWduYXR1cmUnIHBhcmFtZXRlcjoKICAgICAgICBpZiBpbnB1dF9zaWduYXR1cmUgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IHR1cGxlKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRvcmNoLnplcm9zKAogICAgICAgICAgICAgICAgICAgICAgICBzaXplPXNoYXBlLAogICAgICAgICAgICAgICAgICAgICAgICBkdHlwZT1QeVRvcmNoVXRpbHMuY29udmVydF92YWx1ZV90eXBlX3RvX3RvcmNoX2R0eXBlKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICApCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NhbXBsZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICAgICBpbnB1dF9sYXllcnNfbmFtZXM9aW5wdXRfbGF5ZXJzX25hbWVzLAogICAgICAgICAgICBvdXRwdXRfbGF5ZXJzX25hbWVzPW91dHB1dF9sYXllcnNfbmFtZXMsCiAgICAgICAgICAgIGR5bmFtaWNfYXhlcz1keW5hbWljX2F4ZXMsCiAgICAgICAgICAgIGlzX2JhdGNoZWQ9aXNfYmF0Y2hlZAogICAgICAgICkKCgojIE1hcCBmb3IgZ2V0dGluZyB0aGUgY29udmVyc2lvbiBmdW5jdGlvbiBhY2NvcmRpbmcgdG8gdGhlIHByb3ZpZGVkIGZyYW1ld29yazoKX0NPTlZFUlNJT05fTUFQID0gewogICAgInRlbnNvcmZsb3cua2VyYXMiOiBfVG9PTk5YQ29udmVyc2lvbnMudGZfa2VyYXNfdG9fb25ueCwKICAgICJ0b3JjaCI6IF9Ub09OTlhDb252ZXJzaW9ucy5weXRvcmNoX3RvX29ubngsCn0gICMgdHlwZTogRGljdFtzdHIsIENhbGxhYmxlXQoKCmRlZiB0b19vbm54KAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBvbm54X21vZGVsX25hbWU6IHN0ciA9IE5vbmUsCiAgICBvcHRpbWl6ZV9tb2RlbDogYm9vbCA9IFRydWUsCiAgICBmcmFtZXdvcmtfa3dhcmdzOiBEaWN0W3N0ciwgQW55XSA9IE5vbmUsCik6CiAgICAiIiIKICAgIENvbnZlcnQgdGhlIGdpdmVuIG1vZGVsIHRvIGFuIE9OTlggbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgIFRoZSBNTFJ1biBmdW5jdGlvbiBleGVjdXRpb24gY29udGV4dAogICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgIFRoZSBtb2RlbCBwYXRoIHN0b3JlIG9iamVjdC4KICAgIDpwYXJhbSBvbm54X21vZGVsX25hbWU6ICBUaGUgbmFtZSB0byB1c2UgdG8gbG9nIHRoZSBjb252ZXJ0ZWQgT05OWCBtb2RlbC4gSWYgbm90IGdpdmVuLCB0aGUgZ2l2ZW4gYG1vZGVsX25hbWVgIHdpbGwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkIHdpdGggYW4gYWRkaXRpb25hbCBzdWZmaXggYF9vbm54YC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgV2hldGhlciB0byBvcHRpbWl6ZSB0aGUgT05OWCBtb2RlbCB1c2luZyAnb25ueG9wdGltaXplcicgYmVmb3JlIHNhdmluZyB0aGUgbW9kZWwuIERlZmF1bHRlZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRvIFRydWUuCiAgICA6cGFyYW0gZnJhbWV3b3JrX2t3YXJnczogQWRkaXRpb25hbCBhcmd1bWVudHMgZWFjaCBmcmFtZXdvcmsgbWF5IHJlcXVpcmUgaW4gb3JkZXIgdG8gY29udmVydCB0byBPTk5YLiBUbyBnZXQgdGhlIGRvYwogICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0cmluZyBvZiB0aGUgZGVzaXJlZCBmcmFtZXdvcmsgb25ueCBjb252ZXJzaW9uIGZ1bmN0aW9uLCBwYXNzICJoZWxwIi4KICAgICIiIgogICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLmF1dG9fbWxydW4uYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCgogICAgIyBHZXQgYSBtb2RlbCBoYW5kbGVyIG9mIHRoZSByZXF1aXJlZCBmcmFtZXdvcms6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbF9wYXRoLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBHZXQgdGhlIG1vZGVsJ3MgZnJhbWV3b3JrOgogICAgZnJhbWV3b3JrID0gbW9kZWxfaGFuZGxlci5GUkFNRVdPUktfTkFNRQoKICAgICMgVXNlIHRoZSBjb252ZXJzaW9uIG1hcCB0byBnZXQgdGhlIHNwZWNpZmljIGZyYW1ld29yayB0byBvbm54IGNvbnZlcnNpb246CiAgICBpZiBmcmFtZXdvcmsgbm90IGluIF9DT05WRVJTSU9OX01BUDoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJUaGUgZm9sbG93aW5nIGZyYW1ld29yazogJ3tmcmFtZXdvcmt9JywgaGFzIG5vIE9OTlggY29udmVyc2lvbi4iCiAgICAgICAgKQogICAgY29udmVyc2lvbl9mdW5jdGlvbiA9IF9DT05WRVJTSU9OX01BUFtmcmFtZXdvcmtdCgogICAgIyBDaGVjayBpZiBuZWVkZWQgdG8gcHJpbnQgdGhlIGZ1bmN0aW9uJ3MgZG9jIHN0cmluZyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzID09ICJoZWxwIjoKICAgICAgICBwcmludChjb252ZXJzaW9uX2Z1bmN0aW9uLl9fZG9jX18pCiAgICAgICAgcmV0dXJuCgogICAgIyBTZXQgdGhlIGRlZmF1bHQgZW1wdHkgZnJhbWV3b3JrIGt3YXJncyBpZiBuZWVkZWQ6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzIGlzIE5vbmU6CiAgICAgICAgZnJhbWV3b3JrX2t3YXJncyA9IHt9CgogICAgIyBSdW4gdGhlIGNvbnZlcnNpb246CiAgICB0cnk6CiAgICAgICAgY29udmVyc2lvbl9mdW5jdGlvbigKICAgICAgICAgICAgbW9kZWxfaGFuZGxlcj1tb2RlbF9oYW5kbGVyLAogICAgICAgICAgICBvbm54X21vZGVsX25hbWU9b25ueF9tb2RlbF9uYW1lLAogICAgICAgICAgICBvcHRpbWl6ZV9tb2RlbD1vcHRpbWl6ZV9tb2RlbCwKICAgICAgICAgICAgKipmcmFtZXdvcmtfa3dhcmdzLAogICAgICAgICkKICAgIGV4Y2VwdCBUeXBlRXJyb3IgYXMgZXhjZXB0aW9uOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIkVSUk9SOiBBIFR5cGVFcnJvciBleGNlcHRpb24gd2FzIHJhaXNlZCBkdXJpbmcgdGhlIGNvbnZlcnNpb246XG57ZXhjZXB0aW9ufS4gIgogICAgICAgICAgICBmIlBsZWFzZSByZWFkIHRoZSB7ZnJhbWV3b3JrfSBmcmFtZXdvcmsgY29udmVyc2lvbiBmdW5jdGlvbiBkb2Mgc3RyaW5nIGJ5IHBhc3NpbmcgJ2hlbHAnIGluIHRoZSAiCiAgICAgICAgICAgIGYiJ2ZyYW1ld29ya19rd2FyZ3MnIGRpY3Rpb25hcnkgcGFyYW1ldGVyLiIKICAgICAgICApCgoKZGVmIG9wdGltaXplKAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBvcHRpbWl6YXRpb25zOiBMaXN0W3N0cl0gPSBOb25lLAogICAgZml4ZWRfcG9pbnQ6IGJvb2wgPSBGYWxzZSwKICAgIG9wdGltaXplZF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAopOgogICAgIiIiCiAgICBPcHRpbWl6ZSB0aGUgZ2l2ZW4gT05OWCBtb2RlbC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgIFRoZSBNTFJ1biBmdW5jdGlvbiBleGVjdXRpb24gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbF9wYXRoOiAgICAgICAgICAgUGF0aCB0byB0aGUgT05OWCBtb2RlbCBvYmplY3QuCiAgICA6cGFyYW0gb3B0aW1pemF0aW9uczogICAgICAgIExpc3Qgb2YgcG9zc2libGUgb3B0aW1pemF0aW9ucy4gVG8gc2VlIHdoYXQgb3B0aW1pemF0aW9ucyBhcmUgYXZhaWxhYmxlLCBwYXNzICJoZWxwIi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgYWxsIG9mIHRoZSBvcHRpbWl6YXRpb25zIHdpbGwgYmUgdXNlZC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gZml4ZWRfcG9pbnQ6ICAgICAgICAgIE9wdGltaXplIHRoZSB3ZWlnaHRzIHVzaW5nIGZpeGVkIHBvaW50LiBEZWZhdWx0ZWQgdG8gRmFsc2UuCiAgICA6cGFyYW0gb3B0aW1pemVkX21vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBvcHRpbWl6ZWQgbW9kZWwuIElmIE5vbmUsIHRoZSBvcmlnaW5hbCBtb2RlbCB3aWxsIGJlIG92ZXJyaWRkZW4uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgIiIiCiAgICAjIEltcG9ydCB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGltcG9ydCBvbm54b3B0aW1pemVyCiAgICBmcm9tIG1scnVuLmZyYW1ld29ya3Mub25ueCBpbXBvcnQgT05OWE1vZGVsSGFuZGxlcgoKICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIHByaW50IHRoZSBhdmFpbGFibGUgb3B0aW1pemF0aW9ucyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBvcHRpbWl6YXRpb25zID09ICJoZWxwIjoKICAgICAgICBhdmFpbGFibGVfcGFzc2VzID0gIlxuKiAiLmpvaW4ob25ueG9wdGltaXplci5nZXRfYXZhaWxhYmxlX3Bhc3NlcygpKQogICAgICAgIHByaW50KGYiVGhlIGF2YWlsYWJsZSBvcHRpbWl6YXRpb25zIGFyZTpcbioge2F2YWlsYWJsZV9wYXNzZXN9IikKICAgICAgICByZXR1cm4KCiAgICAjIENyZWF0ZSB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBPTk5YTW9kZWxIYW5kbGVyKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWxfcGF0aCwgY29udGV4dD1jb250ZXh0CiAgICApCgogICAgIyBMb2FkIHRoZSBPTk5YIG1vZGVsOgogICAgbW9kZWxfaGFuZGxlci5sb2FkKCkKCiAgICAjIE9wdGltaXplIHRoZSBtb2RlbCB1c2luZyB0aGUgZ2l2ZW4gY29uZmlndXJhdGlvbnM6CiAgICBtb2RlbF9oYW5kbGVyLm9wdGltaXplKG9wdGltaXphdGlvbnM9b3B0aW1pemF0aW9ucywgZml4ZWRfcG9pbnQ9Zml4ZWRfcG9pbnQpCgogICAgIyBSZW5hbWUgaWYgbmVlZGVkOgogICAgaWYgb3B0aW1pemVkX21vZGVsX25hbWUgaXMgbm90IE5vbmU6CiAgICAgICAgbW9kZWxfaGFuZGxlci5zZXRfbW9kZWxfbmFtZShtb2RlbF9uYW1lPW9wdGltaXplZF9tb2RlbF9uYW1lKQoKICAgICMgTG9nIHRoZSBvcHRpbWl6ZWQgbW9kZWw6CiAgICBtb2RlbF9oYW5kbGVyLmxvZygpCg== base_image: mlrun/mlrun commands: [] - code_origin: https://github.com/yonishelach/functions.git#f84b9565a33d8159315992ebba5838d41f6cc112:/Users/Yonatan_Shelach/projects/functions/onnx_utils/onnx_utils.py - origin_filename: /Users/Yonatan_Shelach/projects/functions/onnx_utils/onnx_utils.py + code_origin: '' + origin_filename: '' with_mlrun: false auto_build: true requirements: - - onnx~=1.13.0 - - onnxruntime~=1.14.0 - - onnxoptimizer~=0.3.0 - - onnxmltools~=1.11.0 - - tf2onnx~=1.13.0 + - onnx~=1.15.0 + - onnxruntime~=1.8.1 + - onnxoptimizer~=0.2.0 + - onnxmltools~=1.9.0 + - tf2onnx~=1.16.0 entry_points: tf_keras_to_onnx: name: tf_keras_to_onnx @@ -35,7 +35,6 @@ spec: - name: model_handler doc: An initialized TFKerasModelHandler with a loaded model to convert to ONNX. - default: '' - name: onnx_model_name type: str doc: The name to use to log the converted ONNX model. If not given, the given @@ -55,9 +54,10 @@ spec: data type, a mlrun.data_types.ValueType string. If None, the input signature will be tried to be read from the model artifact. Defaulted to None.' default: null - outputs: - - default: '' + outputs: [] lineno: 26 + has_varargs: false + has_kwargs: false pytorch_to_onnx: name: pytorch_to_onnx doc: Convert a PyTorch model to an ONNX model and log it back to MLRun as a @@ -66,7 +66,6 @@ spec: - name: model_handler doc: An initialized PyTorchModelHandler with a loaded model to convert to ONNX. - default: '' - name: onnx_model_name type: str doc: The name to use to log the converted ONNX model. If not given, the given @@ -114,9 +113,10 @@ spec: doc: Whether to include a batch size as the first axis in every input and output layer. Defaulted to True. Will be ignored if 'dynamic_axes' is provided. default: true - outputs: - - default: '' + outputs: [] lineno: 81 + has_varargs: false + has_kwargs: false to_onnx: name: to_onnx doc: Convert the given model to an ONNX model. @@ -124,11 +124,9 @@ spec: - name: context type: MLClientCtx doc: The MLRun function execution context - default: '' - name: model_path type: str doc: The model path store object. - default: '' - name: onnx_model_name type: str doc: The name to use to log the converted ONNX model. If not given, the given @@ -146,9 +144,10 @@ spec: ONNX. To get the doc string of the desired framework onnx conversion function, pass "help". default: null - outputs: - - default: '' + outputs: [] lineno: 160 + has_varargs: false + has_kwargs: false optimize: name: optimize doc: Optimize the given ONNX model. @@ -156,11 +155,9 @@ spec: - name: context type: MLClientCtx doc: The MLRun function execution context. - default: '' - name: model_path type: str doc: Path to the ONNX model object. - default: '' - name: optimizations type: List[str] doc: List of possible optimizations. To see what optimizations are available, @@ -176,9 +173,10 @@ spec: doc: The name of the optimized model. If None, the original model will be overridden. Defaulted to None. default: null - outputs: - - default: '' + outputs: [] lineno: 219 + has_varargs: false + has_kwargs: false description: ONNX intigration in MLRun, some utils functions for the ONNX framework, optimizing and converting models from different framework to ONNX using MLRun. default_handler: to_onnx diff --git a/onnx_utils/item.yaml b/onnx_utils/item.yaml index 36335837a..84486d9f8 100644 --- a/onnx_utils/item.yaml +++ b/onnx_utils/item.yaml @@ -12,9 +12,9 @@ labels: author: guyl maintainers: [] marketplaceType: '' -mlrunVersion: 1.1.0 +mlrunVersion: 1.6.3 name: onnx_utils -platformVersion: 3.5.0 +platformVersion: 3.5.4 spec: extra_spec: allow_empty_resources: true @@ -26,10 +26,10 @@ spec: image: mlrun/mlrun kind: job requirements: - - onnx~=1.13.0 - - onnxruntime~=1.14.0 - - onnxoptimizer~=0.3.0 - - onnxmltools~=1.11.0 - - tf2onnx~=1.13.0 + - onnx~=1.15.0 + - onnxruntime~=1.8.1 + - onnxoptimizer~=0.2.0 + - onnxmltools~=1.9.0 + - tf2onnx~=1.16.0 url: '' -version: 1.2.0 +version: 1.3.0 diff --git a/onnx_utils/requirements.txt b/onnx_utils/requirements.txt index dc7ff1e7b..a9acd7371 100644 --- a/onnx_utils/requirements.txt +++ b/onnx_utils/requirements.txt @@ -1,11 +1,11 @@ tqdm~=4.62.3 -tensorflow~=2.7.0 +tensorflow~=2.13.0 torch~=1.10.0 torchvision~=0.11.1 -onnx~=1.10.1 -onnxruntime~=1.8.1 -onnxoptimizer~=0.2.0 +onnx~=1.15.0 +onnxruntime~=1.12.1 +onnxoptimizer~=0.3.0 onnxmltools~=1.9.0 -tf2onnx~=1.9.0 +tf2onnx~=1.16.0 plotly~=5.4.0 -wrapt<1.15.0 # wrapt==1.15.0 fails tensorflow 2.7 Todo: please remove when updating tensorflow \ No newline at end of file +#wrapt<1.15.0 # wrapt==1.15.0 fails tensorflow 2.7 Todo: please remove when updating tensorflow \ No newline at end of file diff --git a/onnx_utils/test_onnx_utils.py b/onnx_utils/test_onnx_utils.py index 35b224c4a..aaae96372 100644 --- a/onnx_utils/test_onnx_utils.py +++ b/onnx_utils/test_onnx_utils.py @@ -257,7 +257,7 @@ def test_pytorch_to_onnx(): filename="test_onnx_utils.py", name="log_model", kind="job", - image="mlrun/ml-models", + image="mlrun/mlrun", ) # Run the function to log the model: @@ -341,7 +341,7 @@ def test_optimize(): filename="test_onnx_utils.py", name="log_model", kind="job", - image="mlrun/ml-models", + image="mlrun/mlrun", ) # Run the function to log the model: From 6fdc1b1adb171283d70b2e5458d0c943d93c4f8e Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Thu, 29 Aug 2024 13:25:11 +0300 Subject: [PATCH 2/8] [feature selection] update function yaml --- feature_selection/function.yaml | 75 +++++++++++++-------------------- feature_selection/item.yaml | 6 +-- 2 files changed, 32 insertions(+), 49 deletions(-) diff --git a/feature_selection/function.yaml b/feature_selection/function.yaml index 0851f54d3..d4e95f1e9 100644 --- a/feature_selection/function.yaml +++ b/feature_selection/function.yaml @@ -1,58 +1,33 @@ kind: job -metadata: - name: feature-selection - tag: '' - hash: 6dba16d062d81f78d3d210fee75edfe8b1def9b3 - project: '' - labels: - author: orz - categories: - - data-preparation - - machine-learning +verbose: false spec: + disable_auto_mount: false command: '' - args: [] - image: mlrun/mlrun - build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IGpzb24KaW1wb3J0IG9zCgppbXBvcnQgbWF0cGxvdGxpYi5weXBsb3QgYXMgcGx0CmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgbWxydW4uZmVhdHVyZV9zdG9yZSBhcyBmcwppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgc2VhYm9ybiBhcyBzbnMKZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IFBsb3RBcnRpZmFjdApmcm9tIG1scnVuLmRhdGFzdG9yZS50YXJnZXRzIGltcG9ydCBQYXJxdWV0VGFyZ2V0CiMgTUxSdW4gdXRpbHMKZnJvbSBtbHJ1bi51dGlscy5oZWxwZXJzIGltcG9ydCBjcmVhdGVfY2xhc3MKIyBGZWF0dXJlIHNlbGVjdGlvbiBzdHJhdGVnaWVzCmZyb20gc2tsZWFybi5mZWF0dXJlX3NlbGVjdGlvbiBpbXBvcnQgU2VsZWN0RnJvbU1vZGVsLCBTZWxlY3RLQmVzdAojIFNjYWxlIGZlYXR1cmUgc2NvcmVzZ2l0IHN0CmZyb20gc2tsZWFybi5wcmVwcm9jZXNzaW5nIGltcG9ydCBNaW5NYXhTY2FsZXIKIyBTS0xlYXJuIGVzdGltYXRvcnMgbGlzdApmcm9tIHNrbGVhcm4udXRpbHMgaW1wb3J0IGFsbF9lc3RpbWF0b3JzCgpERUZBVUxUX1NUQVRfRklMVEVSUyA9IFsiZl9jbGFzc2lmIiwgIm11dHVhbF9pbmZvX2NsYXNzaWYiLCAiY2hpMiIsICJmX3JlZ3Jlc3Npb24iXQpERUZBVUxUX01PREVMX0ZJTFRFUlMgPSB7CiAgICAiTGluZWFyU1ZDIjogIkxpbmVhclNWQyIsCiAgICAiTG9naXN0aWNSZWdyZXNzaW9uIjogIkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAiRXh0cmFUcmVlc0NsYXNzaWZpZXIiOiAiRXh0cmFUcmVlc0NsYXNzaWZpZXIiLAp9CgoKZGVmIF9jbGVhcl9jdXJyZW50X2ZpZ3VyZSgpOgogICAgIiIiCiAgICBDbGVhciBtYXRwbG90bGliIGN1cnJlbnQgZmlndXJlLgogICAgIiIiCiAgICBwbHQuY2xhKCkKICAgIHBsdC5jbGYoKQogICAgcGx0LmNsb3NlKCkKCgpkZWYgc2hvd192YWx1ZXNfb25fYmFycyhheHMsIGhfdj0idiIsIHNwYWNlPTAuNCk6CiAgICBkZWYgX3Nob3dfb25fc2luZ2xlX3Bsb3QoYXhfKToKICAgICAgICBpZiBoX3YgPT0gInYiOgogICAgICAgICAgICBmb3IgcCBpbiBheF8ucGF0Y2hlczoKICAgICAgICAgICAgICAgIF94ID0gcC5nZXRfeCgpICsgcC5nZXRfd2lkdGgoKSAvIDIKICAgICAgICAgICAgICAgIF95ID0gcC5nZXRfeSgpICsgcC5nZXRfaGVpZ2h0KCkKICAgICAgICAgICAgICAgIHZhbHVlID0gaW50KHAuZ2V0X2hlaWdodCgpKQogICAgICAgICAgICAgICAgYXhfLnRleHQoX3gsIF95LCB2YWx1ZSwgaGE9ImNlbnRlciIpCiAgICAgICAgZWxpZiBoX3YgPT0gImgiOgogICAgICAgICAgICBmb3IgcCBpbiBheF8ucGF0Y2hlczoKICAgICAgICAgICAgICAgIF94ID0gcC5nZXRfeCgpICsgcC5nZXRfd2lkdGgoKSArIGZsb2F0KHNwYWNlKQogICAgICAgICAgICAgICAgX3kgPSBwLmdldF95KCkgKyBwLmdldF9oZWlnaHQoKQogICAgICAgICAgICAgICAgdmFsdWUgPSBpbnQocC5nZXRfd2lkdGgoKSkKICAgICAgICAgICAgICAgIGF4Xy50ZXh0KF94LCBfeSwgdmFsdWUsIGhhPSJsZWZ0IikKCiAgICBpZiBpc2luc3RhbmNlKGF4cywgbnAubmRhcnJheSk6CiAgICAgICAgZm9yIGlkeCwgYXggaW4gbnAubmRlbnVtZXJhdGUoYXhzKToKICAgICAgICAgICAgX3Nob3dfb25fc2luZ2xlX3Bsb3QoYXgpCiAgICBlbHNlOgogICAgICAgIF9zaG93X29uX3NpbmdsZV9wbG90KGF4cykKCgpkZWYgcGxvdF9zdGF0KGNvbnRleHQsIHN0YXRfbmFtZSwgc3RhdF9kZik6CiAgICBfY2xlYXJfY3VycmVudF9maWd1cmUoKQoKICAgICMgQWRkIGNoYXJ0CiAgICBheCA9IHBsdC5heGVzKCkKICAgIHN0YXRfY2hhcnQgPSBzbnMuYmFycGxvdCgKICAgICAgICB4PXN0YXRfbmFtZSwKICAgICAgICB5PSJpbmRleCIsCiAgICAgICAgZGF0YT1zdGF0X2RmLnNvcnRfdmFsdWVzKHN0YXRfbmFtZSwgYXNjZW5kaW5nPUZhbHNlKS5yZXNldF9pbmRleCgpLAogICAgICAgIGF4PWF4LAogICAgKQogICAgcGx0LnRpZ2h0X2xheW91dCgpCgogICAgZm9yIHAgaW4gc3RhdF9jaGFydC5wYXRjaGVzOgogICAgICAgIHdpZHRoID0gcC5nZXRfd2lkdGgoKQogICAgICAgIHBsdC50ZXh0KAogICAgICAgICAgICA1ICsgcC5nZXRfd2lkdGgoKSwKICAgICAgICAgICAgcC5nZXRfeSgpICsgMC41NSAqIHAuZ2V0X2hlaWdodCgpLAogICAgICAgICAgICAiezoxLjJmfSIuZm9ybWF0KHdpZHRoKSwKICAgICAgICAgICAgaGE9ImNlbnRlciIsCiAgICAgICAgICAgIHZhPSJjZW50ZXIiLAogICAgICAgICkKCiAgICBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBQbG90QXJ0aWZhY3QoZiJ7c3RhdF9uYW1lfSIsIGJvZHk9cGx0LmdjZigpKSwKICAgICAgICBsb2NhbF9wYXRoPW9zLnBhdGguam9pbigicGxvdHMiLCAiZmVhdHVyZV9zZWxlY3Rpb24iLCBmIntzdGF0X25hbWV9Lmh0bWwiKSwKICAgICkKICAgIF9jbGVhcl9jdXJyZW50X2ZpZ3VyZSgpCgoKZGVmIGZlYXR1cmVfc2VsZWN0aW9uKAogICAgY29udGV4dCwKICAgIGRmX2FydGlmYWN0LAogICAgazogaW50ID0gNSwKICAgIG1pbl92b3RlczogZmxvYXQgPSAwLjUsCiAgICBsYWJlbF9jb2x1bW46IHN0ciA9IE5vbmUsCiAgICBzdGF0X2ZpbHRlcnM6IGxpc3QgPSBOb25lLAogICAgbW9kZWxfZmlsdGVyczogZGljdCA9IE5vbmUsCiAgICBtYXhfc2NhbGVkX3Njb3JlczogYm9vbCA9IFRydWUsCiAgICBzYW1wbGVfcmF0aW86IGZsb2F0ID0gTm9uZSwKICAgIG91dHB1dF92ZWN0b3JfbmFtZTogZmxvYXQgPSBOb25lLAogICAgaWdub3JlX3R5cGVfZXJyb3JzOiBib29sID0gRmFsc2UsCiAgICBpc19mZWF0dXJlX3ZlY3RvcjogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBBcHBsaWVzIHNlbGVjdGVkIGZlYXR1cmUgc2VsZWN0aW9uIHN0YXRpc3RpY2FsIGZ1bmN0aW9ucyBvciBtb2RlbHMgb24gb3VyICdkZl9hcnRpZmFjdCcuCgogICAgRWFjaCBzdGF0aXN0aWNhbCBmdW5jdGlvbiBvciBtb2RlbCB3aWxsIHZvdGUgZm9yIGl0J3MgYmVzdCBLIHNlbGVjdGVkIGZlYXR1cmVzLgogICAgSWYgYSBmZWF0dXJlIGhhcyA+PSAnbWluX3ZvdGVzJyB2b3RlcywgaXQgd2lsbCBiZSBzZWxlY3RlZC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgdGhlIGZ1bmN0aW9uIGNvbnRleHQuCiAgICA6cGFyYW0gZGZfYXJ0aWZhY3Q6ICAgICAgICAgZGF0YWZyYW1lIHRvIHBhc3MgYXMgaW5wdXQuCiAgICA6cGFyYW0gazogICAgICAgICAgICAgICAgICAgbnVtYmVyIG9mIHRvcCBmZWF0dXJlcyB0byBzZWxlY3QgZnJvbSBlYWNoIHN0YXRpc3RpY2FsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnVuY3Rpb24gb3IgbW9kZWwuCiAgICA6cGFyYW0gbWluX3ZvdGVzOiAgICAgICAgICAgbWluaW1hbCBudW1iZXIgb2Ygdm90ZXMgKGZyb20gYSBtb2RlbCBvciBieSBzdGF0aXN0aWNhbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZ1bmN0aW9uKSBuZWVkZWQgZm9yIGEgZmVhdHVyZSB0byBiZSBzZWxlY3RlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgc3BlY2lmaWVkIGJ5IHBlcmNlbnRhZ2Ugb2Ygdm90ZXMgb3IgYWJzb2x1dGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBudW1iZXIgb2Ygdm90ZXMuCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uOiAgICAgICAgZ3JvdW5kLXRydXRoICh5KSBsYWJlbHMuCiAgICA6cGFyYW0gc3RhdF9maWx0ZXJzOiAgICAgICAgc3RhdGlzdGljYWwgZnVuY3Rpb25zIHRvIGFwcGx5IHRvIHRoZSBmZWF0dXJlcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChmcm9tIHNrbGVhcm4uZmVhdHVyZV9zZWxlY3Rpb24pLgogICAgOnBhcmFtIG1vZGVsX2ZpbHRlcnM6ICAgICAgIG1vZGVscyB0byB1c2UgZm9yIGZlYXR1cmUgZXZhbHVhdGlvbiwgY2FuIGJlIHNwZWNpZmllZCBieQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsIG5hbWUgKGV4LiBMaW5lYXJTVkMpLCBmb3JtYWxpemVkIGpzb24gKGNvbnRhaW5zICdDTEFTUycsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ0ZJVCcsICdNRVRBJykgb3IgYSBwYXRoIHRvIHN1Y2gganNvbiBmaWxlLgogICAgOnBhcmFtIG1heF9zY2FsZWRfc2NvcmVzOiAgIHByb2R1Y2UgZmVhdHVyZSBzY29yZXMgdGFibGUgc2NhbGVkIHdpdGggbWF4X3NjYWxlci4KICAgIDpwYXJhbSBzYW1wbGVfcmF0aW86ICAgICAgICBwZXJjZW50YWdlIG9mIHRoZSBkYXRhc2V0IHRoZSB1c2VyIHdoaXNoZXMgdG8gY29tcHV0ZSB0aGUgZmVhdHVyZSBzZWxlY3Rpb24gcHJvY2VzcyBvbi4KICAgIDpwYXJhbSBvdXRwdXRfdmVjdG9yX25hbWU6ICBjcmVhdGVzIGEgbmV3IGZlYXR1cmUgdmVjdG9yIGNvbnRhaW5pbmcgb25seSB0aGUgaWRlbnRpZmllcyBmZWF0dXJlcy4KICAgIDpwYXJhbSBpZ25vcmVfdHlwZV9lcnJvcnM6ICBza2lwcyBkYXRhdHlwZXMgdGhhdCBhcmUgbmVpdGhlciBmbG9hdCBub3IgaW50IHdpdGhpbiB0aGUgZmVhdHVyZSB2ZWN0b3IuCiAgICA6cGFyYW0gaXNfZmVhdHVyZV92ZWN0b3I6ICAgYm9vbCBzdGF0aW5nIGlmIHRoZSBkYXRhIGlzIHBhc3NlZCBhcyBhIGZlYXR1cmUgdmVjdG9yLgogICAgIiIiCiAgICBzdGF0X2ZpbHRlcnMgPSBzdGF0X2ZpbHRlcnMgb3IgREVGQVVMVF9TVEFUX0ZJTFRFUlMKICAgIG1vZGVsX2ZpbHRlcnMgPSBtb2RlbF9maWx0ZXJzIG9yIERFRkFVTFRfTU9ERUxfRklMVEVSUwogICAgIyBDaGVjayBpZiBkZi5tZXRhIGlzIHZhbGlkLCBpZiBpdCBpcywgbG9vayBmb3IgYSBmZWF0dXJlIHZlY3RvcgogICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGZfYXJ0aWZhY3QuYXJ0aWZhY3RfdXJsKQogICAgaXNfZmVhdHVyZV92ZWN0b3IgPSBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXgKCiAgICAjIExvb2sgaW5zaWRlIG1ldGEuc3BlYy5sYWJlbF9mZWF0dXJlIHRvIGlkZW50aWZ5IHRoZSBsYWJlbF9jb2x1bW4gaWYgdGhlIHVzZXIgZGlkIG5vdCBzcGVjaWZ5IGl0CiAgICBpZiBsYWJlbF9jb2x1bW4gaXMgTm9uZToKICAgICAgICBpZiBpc19mZWF0dXJlX3ZlY3RvcjoKICAgICAgICAgICAgbGFiZWxfY29sdW1uID0gZGZfYXJ0aWZhY3QubWV0YS5zcGVjLmxhYmVsX2ZlYXR1cmUuc3BsaXQoIi4iKVsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIk5vIGxhYmVsX2NvbHVtbiB3YXMgZ2l2ZW4sIHBsZWFzZSBhZGQgYSBsYWJlbF9jb2x1bW4uIikKCiAgICAjIFVzZSB0aGUgZmVhdHVyZSB2ZWN0b3IgYXMgZGF0YWZyYW1lCiAgICBkZiA9IGRmX2FydGlmYWN0LmFzX2RmKCkKCiAgICAjIEVuc3VyZSBrIGlzIG5vdCBiaWdnZXIgdGhhbiB0aGUgdG90YWwgbnVtYmVyIG9mIGZlYXR1cmVzCiAgICBpZiBrID4gZGYuc2hhcGVbMV06CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJLIGNhbm5vdCBiZSBiaWdnZXIgdGhhbiB0aGUgdG90YWwgbnVtYmVyIG9mIGZlYXR1cmVzICh7ZGYuc2hhcGVbMV19KS4gUGxlYXNlIGNob29zZSBhIHNtYWxsZXIgSy4iCiAgICAgICAgKQogICAgZWxpZiBrIDwgMToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJLIGNhbm5vdCBiZSBzbWFsbGVyIHRoYW4gMS4gUGxlYXNlIGNob29zZSBhIGJpZ2dlciBLLiIpCgogICAgIyBDcmVhdGUgYSBzYW1wbGUgZGF0YWZyYW1lIG9mIHRoZSBvcmlnaW5hbCBmZWF0dXJlIHZlY3RvcgogICAgaWYgc2FtcGxlX3JhdGlvOgogICAgICAgIGRmID0gKAogICAgICAgICAgICBkZi5ncm91cGJ5KGxhYmVsX2NvbHVtbikKICAgICAgICAgICAgLmFwcGx5KGxhbWJkYSB4OiB4LnNhbXBsZShmcmFjPXNhbXBsZV9yYXRpbykpCiAgICAgICAgICAgIC5yZXNldF9pbmRleChkcm9wPVRydWUpCiAgICAgICAgKQogICAgICAgIGRmID0gZGYuZHJvcG5hKCkKCiAgICAjIFNldCBmZWF0dXJlIHZlY3RvciBhbmQgbGFiZWxzCiAgICB5ID0gZGYucG9wKGxhYmVsX2NvbHVtbikKICAgIFggPSBkZgoKICAgIGlmIG5wLm9iamVjdF8gaW4gbGlzdChYLmR0eXBlcykgYW5kIGlnbm9yZV90eXBlX2Vycm9ycyBpcyBGYWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIntkZi5zZWxlY3RfZHR5cGVzKGluY2x1ZGU9WydvYmplY3QnXSkuY29sdW1ucy50b2xpc3QoKX0gYXJlIG5laXRoZXIgZmxvYXQgb3IgaW50LiIKICAgICAgICApCgogICAgIyBDcmVhdGUgc2VsZWN0ZWQgc3RhdGlzdGljYWwgZXN0aW1hdG9ycwogICAgc3RhdF9mdW5jdGlvbnNfbGlzdCA9IHsKICAgICAgICBzdGF0X25hbWU6IFNlbGVjdEtCZXN0KAogICAgICAgICAgICBzY29yZV9mdW5jPWNyZWF0ZV9jbGFzcyhmInNrbGVhcm4uZmVhdHVyZV9zZWxlY3Rpb24ue3N0YXRfbmFtZX0iKSwgaz1rCiAgICAgICAgKQogICAgICAgIGZvciBzdGF0X25hbWUgaW4gc3RhdF9maWx0ZXJzCiAgICB9CiAgICByZXF1aXJlc19hYnMgPSBbImNoaTIiXQoKICAgICMgUnVuIHN0YXRpc3RpYyBmaWx0ZXJzCiAgICBzZWxlY3RlZF9mZWF0dXJlc19hZ2cgPSB7fQogICAgc3RhdHNfZGYgPSBwZC5EYXRhRnJhbWUoaW5kZXg9WC5jb2x1bW5zKS5kcm9wbmEoKQoKICAgIGZvciBzdGF0X25hbWUsIHN0YXRfZnVuYyBpbiBzdGF0X2Z1bmN0aW9uc19saXN0Lml0ZW1zKCk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICBwYXJhbXMgPSAoWCwgeSkgaWYgc3RhdF9uYW1lIGluIHJlcXVpcmVzX2FicyBlbHNlIChhYnMoWCksIHkpCiAgICAgICAgICAgIHN0YXQgPSBzdGF0X2Z1bmMuZml0KCpwYXJhbXMpCgogICAgICAgICAgICAjIENvbGxlY3Qgc3RhdCBmdW5jdGlvbiByZXN1bHRzCiAgICAgICAgICAgIHN0YXRfZGYgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgICAgICAgICBpbmRleD1YLmNvbHVtbnMsIGNvbHVtbnM9W3N0YXRfbmFtZV0sIGRhdGE9c3RhdC5zY29yZXNfCiAgICAgICAgICAgICkKICAgICAgICAgICAgcGxvdF9zdGF0KGNvbnRleHQsIHN0YXRfbmFtZSwgc3RhdF9kZikKICAgICAgICAgICAgc3RhdHNfZGYgPSBzdGF0c19kZi5qb2luKHN0YXRfZGYpCgogICAgICAgICAgICAjIFNlbGVjdCBLIEJlc3QgZmVhdHVyZXMKICAgICAgICAgICAgc2VsZWN0ZWRfZmVhdHVyZXMgPSBYLmNvbHVtbnNbc3RhdF9mdW5jLmdldF9zdXBwb3J0KCldCiAgICAgICAgICAgIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZ1tzdGF0X25hbWVdID0gc2VsZWN0ZWRfZmVhdHVyZXMKCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiQ291bGRuJ3QgY2FsY3VsYXRlIHtzdGF0X25hbWV9IGJlY2F1c2Ugb2Y6IHtlfSIpCgogICAgIyBDcmVhdGUgbW9kZWxzIGZyb20gY2xhc3MgbmFtZSAvIGpzb24gZmlsZSAvIGpzb24gcGFyYW1zCiAgICBhbGxfc2tsZWFybl9lc3RpbWF0b3JzID0gZGljdChhbGxfZXN0aW1hdG9ycygpKSBpZiBsZW4obW9kZWxfZmlsdGVycykgPiAwIGVsc2Uge30KICAgIHNlbGVjdGVkX21vZGVscyA9IHt9CiAgICBmb3IgbW9kZWxfbmFtZSwgbW9kZWwgaW4gbW9kZWxfZmlsdGVycy5pdGVtcygpOgogICAgICAgIGlmICIuanNvbiIgaW4gbW9kZWw6CiAgICAgICAgICAgIGN1cnJlbnRfbW9kZWwgPSBqc29uLmxvYWQob3Blbihtb2RlbCwgInIiKSkKICAgICAgICAgICAgY2xhc3NpZmllcl9jbGFzcyA9IGNyZWF0ZV9jbGFzcyhjdXJyZW50X21vZGVsWyJNRVRBIl1bImNsYXNzIl0pCiAgICAgICAgICAgIHNlbGVjdGVkX21vZGVsc1ttb2RlbF9uYW1lXSA9IGNsYXNzaWZpZXJfY2xhc3MoKipjdXJyZW50X21vZGVsWyJDTEFTUyJdKQogICAgICAgIGVsaWYgbW9kZWwgaW4gYWxsX3NrbGVhcm5fZXN0aW1hdG9yczoKICAgICAgICAgICAgc2VsZWN0ZWRfbW9kZWxzW21vZGVsX25hbWVdID0gYWxsX3NrbGVhcm5fZXN0aW1hdG9yc1ttb2RlbF9uYW1lXSgpCgogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGN1cnJlbnRfbW9kZWwgPSBqc29uLmxvYWRzKG1vZGVsKQogICAgICAgICAgICAgICAgY2xhc3NpZmllcl9jbGFzcyA9IGNyZWF0ZV9jbGFzcyhjdXJyZW50X21vZGVsWyJNRVRBIl1bImNsYXNzIl0pCiAgICAgICAgICAgICAgICBzZWxlY3RlZF9tb2RlbHNbbW9kZWxfbmFtZV0gPSBjbGFzc2lmaWVyX2NsYXNzKCoqY3VycmVudF9tb2RlbFsiQ0xBU1MiXSkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInVuYWJsZSB0byBsb2FkIHttb2RlbH0gYmVjYXVzZSBvZjoge2V9IikKCiAgICAjIFJ1biBtb2RlbCBmaWx0ZXJzCiAgICBtb2RlbHNfZGYgPSBwZC5EYXRhRnJhbWUoaW5kZXg9WC5jb2x1bW5zKQogICAgZm9yIG1vZGVsX25hbWUsIG1vZGVsIGluIHNlbGVjdGVkX21vZGVscy5pdGVtcygpOgoKICAgICAgICBpZiBtb2RlbF9uYW1lID09ICJMb2dpc3RpY1JlZ3Jlc3Npb24iOgogICAgICAgICAgICBtb2RlbC5zZXRfcGFyYW1zKHNvbHZlcj0ibGlibGluZWFyIikKCiAgICAgICAgIyBUcmFpbiBtb2RlbCBhbmQgZ2V0IGZlYXR1cmUgaW1wb3J0YW5jZQogICAgICAgIHNlbGVjdF9mcm9tX21vZGVsID0gU2VsZWN0RnJvbU1vZGVsKG1vZGVsKS5maXQoWCwgeSkKICAgICAgICBmZWF0dXJlX2lkeCA9IHNlbGVjdF9mcm9tX21vZGVsLmdldF9zdXBwb3J0KCkKICAgICAgICBmZWF0dXJlX25hbWVzID0gWC5jb2x1bW5zW2ZlYXR1cmVfaWR4XQogICAgICAgIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZ1ttb2RlbF9uYW1lXSA9IGZlYXR1cmVfbmFtZXMudG9saXN0KCkKCiAgICAgICAgIyBDb2xsZWN0IG1vZGVsIGZlYXR1cmUgaW1wb3J0YW5jZQogICAgICAgIGlmIGhhc2F0dHIoc2VsZWN0X2Zyb21fbW9kZWwuZXN0aW1hdG9yXywgImNvZWZfIik6CiAgICAgICAgICAgIHN0YXRfZGYgPSBzZWxlY3RfZnJvbV9tb2RlbC5lc3RpbWF0b3JfLmNvZWZfCiAgICAgICAgZWxpZiBoYXNhdHRyKHNlbGVjdF9mcm9tX21vZGVsLmVzdGltYXRvcl8sICJmZWF0dXJlX2ltcG9ydGFuY2VzXyIpOgogICAgICAgICAgICBzdGF0X2RmID0gc2VsZWN0X2Zyb21fbW9kZWwuZXN0aW1hdG9yXy5mZWF0dXJlX2ltcG9ydGFuY2VzXwoKICAgICAgICBzdGF0X2RmID0gcGQuRGF0YUZyYW1lKGluZGV4PVguY29sdW1ucywgY29sdW1ucz1bbW9kZWxfbmFtZV0sIGRhdGE9c3RhdF9kZlswXSkKICAgICAgICBtb2RlbHNfZGYgPSBtb2RlbHNfZGYuam9pbihzdGF0X2RmKQoKICAgICAgICBwbG90X3N0YXQoY29udGV4dCwgbW9kZWxfbmFtZSwgc3RhdF9kZikKCiAgICAjIENyZWF0ZSBmZWF0dXJlX3Njb3JlcyBERiB3aXRoIHN0YXQgJiBtb2RlbCBmaWx0ZXJzIHNjb3JlcwogICAgcmVzdWx0X21hdHJpeF9kZiA9IHBkLmNvbmNhdChbc3RhdHNfZGYsIG1vZGVsc19kZl0sIGF4aXM9MSwgc29ydD1GYWxzZSkKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoCiAgICAgICAga2V5PSJmZWF0dXJlX3Njb3JlcyIsCiAgICAgICAgZGY9cmVzdWx0X21hdHJpeF9kZiwKICAgICAgICBsb2NhbF9wYXRoPSJmZWF0dXJlX3Njb3Jlcy5wYXJxdWV0IiwKICAgICAgICBmb3JtYXQ9InBhcnF1ZXQiLAogICAgKQogICAgaWYgbWF4X3NjYWxlZF9zY29yZXM6CiAgICAgICAgbm9ybWFsaXplZF9kZiA9IHJlc3VsdF9tYXRyaXhfZGYucmVwbGFjZShbbnAuaW5mLCAtbnAuaW5mXSwgbnAubmFuKS52YWx1ZXMKICAgICAgICBtaW5fbWF4X3NjYWxlciA9IE1pbk1heFNjYWxlcigpCiAgICAgICAgbm9ybWFsaXplZF9kZiA9IG1pbl9tYXhfc2NhbGVyLmZpdF90cmFuc2Zvcm0obm9ybWFsaXplZF9kZikKICAgICAgICBub3JtYWxpemVkX2RmID0gcGQuRGF0YUZyYW1lKAogICAgICAgICAgICBkYXRhPW5vcm1hbGl6ZWRfZGYsCiAgICAgICAgICAgIGNvbHVtbnM9cmVzdWx0X21hdHJpeF9kZi5jb2x1bW5zLAogICAgICAgICAgICBpbmRleD1yZXN1bHRfbWF0cml4X2RmLmluZGV4LAogICAgICAgICkKICAgICAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgICAgICBrZXk9Im1heF9zY2FsZWRfc2NvcmVzX2ZlYXR1cmVfc2NvcmVzIiwKICAgICAgICAgICAgZGY9bm9ybWFsaXplZF9kZiwKICAgICAgICAgICAgbG9jYWxfcGF0aD0ibWF4X3NjYWxlZF9zY29yZXNfZmVhdHVyZV9zY29yZXMucGFycXVldCIsCiAgICAgICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICAgICAgKQoKICAgICMgQ3JlYXRlIGZlYXR1cmUgY291bnQgRGF0YUZyYW1lCiAgICBmb3IgdGVzdF9uYW1lIGluIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZzoKICAgICAgICByZXN1bHRfbWF0cml4X2RmW3Rlc3RfbmFtZV0gPSBbCiAgICAgICAgICAgIDEgaWYgeCBpbiBzZWxlY3RlZF9mZWF0dXJlc19hZ2dbdGVzdF9uYW1lXSBlbHNlIDAgZm9yIHggaW4gWC5jb2x1bW5zCiAgICAgICAgXQogICAgcmVzdWx0X21hdHJpeF9kZi5sb2NbOiwgIm51bV92b3RlcyJdID0gcmVzdWx0X21hdHJpeF9kZi5zdW0oYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICBrZXk9InNlbGVjdGVkX2ZlYXR1cmVzX2NvdW50IiwKICAgICAgICBkZj1yZXN1bHRfbWF0cml4X2RmLAogICAgICAgIGxvY2FsX3BhdGg9InNlbGVjdGVkX2ZlYXR1cmVzX2NvdW50LnBhcnF1ZXQiLAogICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICApCgogICAgIyBIb3cgbWFueSB2b3RlcyBhcmUgbmVlZGVkIGZvciBhIGZlYXR1cmUgdG8gYmUgc2VsZWN0ZWQ/CiAgICBpZiBpc2luc3RhbmNlKG1pbl92b3RlcywgaW50KToKICAgICAgICB2b3Rlc19uZWVkZWQgPSBtaW5fdm90ZXMKICAgIGVsc2U6CiAgICAgICAgbnVtX2ZpbHRlcnMgPSBsZW4oc3RhdF9maWx0ZXJzKSArIGxlbihtb2RlbF9maWx0ZXJzKQogICAgICAgIHZvdGVzX25lZWRlZCA9IGludChucC5mbG9vcihudW1fZmlsdGVycyAqIG1heChtaW4obWluX3ZvdGVzLCAxKSwgMCkpKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInZvdGVzIG5lZWRlZCB0byBiZSBzZWxlY3RlZDoge3ZvdGVzX25lZWRlZH0iKQoKICAgICMgQ3JlYXRlIGZpbmFsIGZlYXR1cmUgZGF0YWZyYW1lCiAgICBzZWxlY3RlZF9mZWF0dXJlcyA9IHJlc3VsdF9tYXRyaXhfZGZbCiAgICAgICAgcmVzdWx0X21hdHJpeF9kZi5udW1fdm90ZXMgPj0gdm90ZXNfbmVlZGVkCiAgICBdLmluZGV4LnRvbGlzdCgpCiAgICBnb29kX2ZlYXR1cmVfZGYgPSBkZi5sb2NbOiwgc2VsZWN0ZWRfZmVhdHVyZXNdCiAgICBmaW5hbF9kZiA9IHBkLmNvbmNhdChbZ29vZF9mZWF0dXJlX2RmLCB5XSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICBrZXk9InNlbGVjdGVkX2ZlYXR1cmVzIiwKICAgICAgICBkZj1maW5hbF9kZiwKICAgICAgICBsb2NhbF9wYXRoPSJzZWxlY3RlZF9mZWF0dXJlcy5wYXJxdWV0IiwKICAgICAgICBmb3JtYXQ9InBhcnF1ZXQiLAogICAgKQoKICAgICMgQ3JlYXRpbmcgYSBuZXcgZmVhdHVyZSB2ZWN0b3IgY29udGFpbmluZyBvbmx5IHRoZSBpZGVudGlmaWVkIHRvcCBmZWF0dXJlcwogICAgaWYgaXNfZmVhdHVyZV92ZWN0b3IgYW5kIGRmX2FydGlmYWN0Lm1ldGEuc3BlYy5mZWF0dXJlcyBhbmQgb3V0cHV0X3ZlY3Rvcl9uYW1lOgogICAgICAgICMgU2VsZWN0aW5nIHRoZSB0b3AgSyBmZWF0dXJlcyBmcm9tIG91ciB0b3AgZmVhdHVyZSBkYXRhZnJhbWUKICAgICAgICBzZWxlY3RlZF9mZWF0dXJlcyA9IHJlc3VsdF9tYXRyaXhfZGYuaGVhZChrKS5pbmRleAoKICAgICAgICAjIE1hdGNoIHRoZSBzZWxlY3RlZCBmZWF0dXJlIG5hbWVzIHRvIHRoZSBGUyBGZWF0dXJlIGFubm90YXRpb25zCiAgICAgICAgbWF0Y2hlZF9zZWxlY3Rpb25zID0gWwogICAgICAgICAgICBmZWF0dXJlCiAgICAgICAgICAgIGZvciBmZWF0dXJlIGluIGxpc3QoZGZfYXJ0aWZhY3QubWV0YS5zcGVjLmZlYXR1cmVzKQogICAgICAgICAgICBmb3Igc2VsZWN0ZWQgaW4gbGlzdChzZWxlY3RlZF9mZWF0dXJlcykKICAgICAgICAgICAgaWYgZmVhdHVyZS5lbmRzd2l0aChzZWxlY3RlZCkKICAgICAgICBdCgogICAgICAgICMgRGVmaW5pbmcgb3VyIG5ldyBmZWF0dXJlIHZlY3RvcgogICAgICAgIHRvcF9mZWF0dXJlc19mdiA9IGZzLkZlYXR1cmVWZWN0b3IoCiAgICAgICAgICAgIG91dHB1dF92ZWN0b3JfbmFtZSwKICAgICAgICAgICAgbWF0Y2hlZF9zZWxlY3Rpb25zLAogICAgICAgICAgICBsYWJlbF9mZWF0dXJlPSJsYWJlbHMubGFiZWwiLAogICAgICAgICAgICBkZXNjcmlwdGlvbj0iZmVhdHVyZSB2ZWN0b3IgY29tcG9zZWQgc3RyaWN0bHkgb2Ygb3VyIHRvcCBmZWF0dXJlcyIsCiAgICAgICAgKQoKICAgICAgICAjIFNhdmluZwogICAgICAgIHRvcF9mZWF0dXJlc19mdi5zYXZlKCkKICAgICAgICBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcyh0b3BfZmVhdHVyZXNfZnYsIHRhcmdldD1QYXJxdWV0VGFyZ2V0KCkpCgogICAgICAgICMgTG9nZ2luZyBvdXIgbmV3IGZlYXR1cmUgdmVjdG9yIFVSSQogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdCgidG9wX2ZlYXR1cmVzX3ZlY3RvciIsIHRvcF9mZWF0dXJlc19mdi51cmkpCg== - commands: [] - code_origin: '' - origin_filename: '' - requirements: [] entry_points: show_values_on_bars: - name: show_values_on_bars - doc: '' + lineno: 54 parameters: - name: axs - name: h_v default: v - name: space default: 0.4 - outputs: [] - lineno: 54 + name: show_values_on_bars has_varargs: false has_kwargs: false - plot_stat: - name: plot_stat doc: '' + plot_stat: + lineno: 76 parameters: - name: context - name: stat_name - name: stat_df - outputs: [] - lineno: 76 + name: plot_stat has_varargs: false has_kwargs: false + doc: '' feature_selection: - name: feature_selection - doc: 'Applies selected feature selection statistical functions or models on - our ''df_artifact''. - - - Each statistical function or model will vote for it''s best K selected features. - - If a feature has >= ''min_votes'' votes, it will be selected.' + lineno: 106 parameters: - name: context doc: the function context. @@ -103,18 +78,26 @@ spec: type: bool doc: bool stating if the data is passed as a feature vector. default: false - outputs: [] - lineno: 106 + name: feature_selection has_varargs: false has_kwargs: false - description: Select features through multiple Statistical and Model filters + doc: 'Applies selected feature selection statistical functions or models on + our ''df_artifact''. + + + Each statistical function or model will vote for it''s best K selected features. + + If a feature has >= ''min_votes'' votes, it will be selected.' + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IGpzb24KaW1wb3J0IG9zCgppbXBvcnQgbWF0cGxvdGxpYi5weXBsb3QgYXMgcGx0CmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgbWxydW4uZmVhdHVyZV9zdG9yZSBhcyBmcwppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgc2VhYm9ybiBhcyBzbnMKZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IFBsb3RBcnRpZmFjdApmcm9tIG1scnVuLmRhdGFzdG9yZS50YXJnZXRzIGltcG9ydCBQYXJxdWV0VGFyZ2V0CiMgTUxSdW4gdXRpbHMKZnJvbSBtbHJ1bi51dGlscy5oZWxwZXJzIGltcG9ydCBjcmVhdGVfY2xhc3MKIyBGZWF0dXJlIHNlbGVjdGlvbiBzdHJhdGVnaWVzCmZyb20gc2tsZWFybi5mZWF0dXJlX3NlbGVjdGlvbiBpbXBvcnQgU2VsZWN0RnJvbU1vZGVsLCBTZWxlY3RLQmVzdAojIFNjYWxlIGZlYXR1cmUgc2NvcmVzZ2l0IHN0CmZyb20gc2tsZWFybi5wcmVwcm9jZXNzaW5nIGltcG9ydCBNaW5NYXhTY2FsZXIKIyBTS0xlYXJuIGVzdGltYXRvcnMgbGlzdApmcm9tIHNrbGVhcm4udXRpbHMgaW1wb3J0IGFsbF9lc3RpbWF0b3JzCgpERUZBVUxUX1NUQVRfRklMVEVSUyA9IFsiZl9jbGFzc2lmIiwgIm11dHVhbF9pbmZvX2NsYXNzaWYiLCAiY2hpMiIsICJmX3JlZ3Jlc3Npb24iXQpERUZBVUxUX01PREVMX0ZJTFRFUlMgPSB7CiAgICAiTGluZWFyU1ZDIjogIkxpbmVhclNWQyIsCiAgICAiTG9naXN0aWNSZWdyZXNzaW9uIjogIkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAiRXh0cmFUcmVlc0NsYXNzaWZpZXIiOiAiRXh0cmFUcmVlc0NsYXNzaWZpZXIiLAp9CgoKZGVmIF9jbGVhcl9jdXJyZW50X2ZpZ3VyZSgpOgogICAgIiIiCiAgICBDbGVhciBtYXRwbG90bGliIGN1cnJlbnQgZmlndXJlLgogICAgIiIiCiAgICBwbHQuY2xhKCkKICAgIHBsdC5jbGYoKQogICAgcGx0LmNsb3NlKCkKCgpkZWYgc2hvd192YWx1ZXNfb25fYmFycyhheHMsIGhfdj0idiIsIHNwYWNlPTAuNCk6CiAgICBkZWYgX3Nob3dfb25fc2luZ2xlX3Bsb3QoYXhfKToKICAgICAgICBpZiBoX3YgPT0gInYiOgogICAgICAgICAgICBmb3IgcCBpbiBheF8ucGF0Y2hlczoKICAgICAgICAgICAgICAgIF94ID0gcC5nZXRfeCgpICsgcC5nZXRfd2lkdGgoKSAvIDIKICAgICAgICAgICAgICAgIF95ID0gcC5nZXRfeSgpICsgcC5nZXRfaGVpZ2h0KCkKICAgICAgICAgICAgICAgIHZhbHVlID0gaW50KHAuZ2V0X2hlaWdodCgpKQogICAgICAgICAgICAgICAgYXhfLnRleHQoX3gsIF95LCB2YWx1ZSwgaGE9ImNlbnRlciIpCiAgICAgICAgZWxpZiBoX3YgPT0gImgiOgogICAgICAgICAgICBmb3IgcCBpbiBheF8ucGF0Y2hlczoKICAgICAgICAgICAgICAgIF94ID0gcC5nZXRfeCgpICsgcC5nZXRfd2lkdGgoKSArIGZsb2F0KHNwYWNlKQogICAgICAgICAgICAgICAgX3kgPSBwLmdldF95KCkgKyBwLmdldF9oZWlnaHQoKQogICAgICAgICAgICAgICAgdmFsdWUgPSBpbnQocC5nZXRfd2lkdGgoKSkKICAgICAgICAgICAgICAgIGF4Xy50ZXh0KF94LCBfeSwgdmFsdWUsIGhhPSJsZWZ0IikKCiAgICBpZiBpc2luc3RhbmNlKGF4cywgbnAubmRhcnJheSk6CiAgICAgICAgZm9yIGlkeCwgYXggaW4gbnAubmRlbnVtZXJhdGUoYXhzKToKICAgICAgICAgICAgX3Nob3dfb25fc2luZ2xlX3Bsb3QoYXgpCiAgICBlbHNlOgogICAgICAgIF9zaG93X29uX3NpbmdsZV9wbG90KGF4cykKCgpkZWYgcGxvdF9zdGF0KGNvbnRleHQsIHN0YXRfbmFtZSwgc3RhdF9kZik6CiAgICBfY2xlYXJfY3VycmVudF9maWd1cmUoKQoKICAgICMgQWRkIGNoYXJ0CiAgICBheCA9IHBsdC5heGVzKCkKICAgIHN0YXRfY2hhcnQgPSBzbnMuYmFycGxvdCgKICAgICAgICB4PXN0YXRfbmFtZSwKICAgICAgICB5PSJpbmRleCIsCiAgICAgICAgZGF0YT1zdGF0X2RmLnNvcnRfdmFsdWVzKHN0YXRfbmFtZSwgYXNjZW5kaW5nPUZhbHNlKS5yZXNldF9pbmRleCgpLAogICAgICAgIGF4PWF4LAogICAgKQogICAgcGx0LnRpZ2h0X2xheW91dCgpCgogICAgZm9yIHAgaW4gc3RhdF9jaGFydC5wYXRjaGVzOgogICAgICAgIHdpZHRoID0gcC5nZXRfd2lkdGgoKQogICAgICAgIHBsdC50ZXh0KAogICAgICAgICAgICA1ICsgcC5nZXRfd2lkdGgoKSwKICAgICAgICAgICAgcC5nZXRfeSgpICsgMC41NSAqIHAuZ2V0X2hlaWdodCgpLAogICAgICAgICAgICAiezoxLjJmfSIuZm9ybWF0KHdpZHRoKSwKICAgICAgICAgICAgaGE9ImNlbnRlciIsCiAgICAgICAgICAgIHZhPSJjZW50ZXIiLAogICAgICAgICkKCiAgICBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBQbG90QXJ0aWZhY3QoZiJ7c3RhdF9uYW1lfSIsIGJvZHk9cGx0LmdjZigpKSwKICAgICAgICBsb2NhbF9wYXRoPW9zLnBhdGguam9pbigicGxvdHMiLCAiZmVhdHVyZV9zZWxlY3Rpb24iLCBmIntzdGF0X25hbWV9Lmh0bWwiKSwKICAgICkKICAgIF9jbGVhcl9jdXJyZW50X2ZpZ3VyZSgpCgoKZGVmIGZlYXR1cmVfc2VsZWN0aW9uKAogICAgY29udGV4dCwKICAgIGRmX2FydGlmYWN0LAogICAgazogaW50ID0gNSwKICAgIG1pbl92b3RlczogZmxvYXQgPSAwLjUsCiAgICBsYWJlbF9jb2x1bW46IHN0ciA9IE5vbmUsCiAgICBzdGF0X2ZpbHRlcnM6IGxpc3QgPSBOb25lLAogICAgbW9kZWxfZmlsdGVyczogZGljdCA9IE5vbmUsCiAgICBtYXhfc2NhbGVkX3Njb3JlczogYm9vbCA9IFRydWUsCiAgICBzYW1wbGVfcmF0aW86IGZsb2F0ID0gTm9uZSwKICAgIG91dHB1dF92ZWN0b3JfbmFtZTogZmxvYXQgPSBOb25lLAogICAgaWdub3JlX3R5cGVfZXJyb3JzOiBib29sID0gRmFsc2UsCiAgICBpc19mZWF0dXJlX3ZlY3RvcjogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBBcHBsaWVzIHNlbGVjdGVkIGZlYXR1cmUgc2VsZWN0aW9uIHN0YXRpc3RpY2FsIGZ1bmN0aW9ucyBvciBtb2RlbHMgb24gb3VyICdkZl9hcnRpZmFjdCcuCgogICAgRWFjaCBzdGF0aXN0aWNhbCBmdW5jdGlvbiBvciBtb2RlbCB3aWxsIHZvdGUgZm9yIGl0J3MgYmVzdCBLIHNlbGVjdGVkIGZlYXR1cmVzLgogICAgSWYgYSBmZWF0dXJlIGhhcyA+PSAnbWluX3ZvdGVzJyB2b3RlcywgaXQgd2lsbCBiZSBzZWxlY3RlZC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgdGhlIGZ1bmN0aW9uIGNvbnRleHQuCiAgICA6cGFyYW0gZGZfYXJ0aWZhY3Q6ICAgICAgICAgZGF0YWZyYW1lIHRvIHBhc3MgYXMgaW5wdXQuCiAgICA6cGFyYW0gazogICAgICAgICAgICAgICAgICAgbnVtYmVyIG9mIHRvcCBmZWF0dXJlcyB0byBzZWxlY3QgZnJvbSBlYWNoIHN0YXRpc3RpY2FsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnVuY3Rpb24gb3IgbW9kZWwuCiAgICA6cGFyYW0gbWluX3ZvdGVzOiAgICAgICAgICAgbWluaW1hbCBudW1iZXIgb2Ygdm90ZXMgKGZyb20gYSBtb2RlbCBvciBieSBzdGF0aXN0aWNhbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZ1bmN0aW9uKSBuZWVkZWQgZm9yIGEgZmVhdHVyZSB0byBiZSBzZWxlY3RlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgc3BlY2lmaWVkIGJ5IHBlcmNlbnRhZ2Ugb2Ygdm90ZXMgb3IgYWJzb2x1dGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBudW1iZXIgb2Ygdm90ZXMuCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uOiAgICAgICAgZ3JvdW5kLXRydXRoICh5KSBsYWJlbHMuCiAgICA6cGFyYW0gc3RhdF9maWx0ZXJzOiAgICAgICAgc3RhdGlzdGljYWwgZnVuY3Rpb25zIHRvIGFwcGx5IHRvIHRoZSBmZWF0dXJlcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChmcm9tIHNrbGVhcm4uZmVhdHVyZV9zZWxlY3Rpb24pLgogICAgOnBhcmFtIG1vZGVsX2ZpbHRlcnM6ICAgICAgIG1vZGVscyB0byB1c2UgZm9yIGZlYXR1cmUgZXZhbHVhdGlvbiwgY2FuIGJlIHNwZWNpZmllZCBieQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsIG5hbWUgKGV4LiBMaW5lYXJTVkMpLCBmb3JtYWxpemVkIGpzb24gKGNvbnRhaW5zICdDTEFTUycsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ0ZJVCcsICdNRVRBJykgb3IgYSBwYXRoIHRvIHN1Y2gganNvbiBmaWxlLgogICAgOnBhcmFtIG1heF9zY2FsZWRfc2NvcmVzOiAgIHByb2R1Y2UgZmVhdHVyZSBzY29yZXMgdGFibGUgc2NhbGVkIHdpdGggbWF4X3NjYWxlci4KICAgIDpwYXJhbSBzYW1wbGVfcmF0aW86ICAgICAgICBwZXJjZW50YWdlIG9mIHRoZSBkYXRhc2V0IHRoZSB1c2VyIHdoaXNoZXMgdG8gY29tcHV0ZSB0aGUgZmVhdHVyZSBzZWxlY3Rpb24gcHJvY2VzcyBvbi4KICAgIDpwYXJhbSBvdXRwdXRfdmVjdG9yX25hbWU6ICBjcmVhdGVzIGEgbmV3IGZlYXR1cmUgdmVjdG9yIGNvbnRhaW5pbmcgb25seSB0aGUgaWRlbnRpZmllcyBmZWF0dXJlcy4KICAgIDpwYXJhbSBpZ25vcmVfdHlwZV9lcnJvcnM6ICBza2lwcyBkYXRhdHlwZXMgdGhhdCBhcmUgbmVpdGhlciBmbG9hdCBub3IgaW50IHdpdGhpbiB0aGUgZmVhdHVyZSB2ZWN0b3IuCiAgICA6cGFyYW0gaXNfZmVhdHVyZV92ZWN0b3I6ICAgYm9vbCBzdGF0aW5nIGlmIHRoZSBkYXRhIGlzIHBhc3NlZCBhcyBhIGZlYXR1cmUgdmVjdG9yLgogICAgIiIiCiAgICBzdGF0X2ZpbHRlcnMgPSBzdGF0X2ZpbHRlcnMgb3IgREVGQVVMVF9TVEFUX0ZJTFRFUlMKICAgIG1vZGVsX2ZpbHRlcnMgPSBtb2RlbF9maWx0ZXJzIG9yIERFRkFVTFRfTU9ERUxfRklMVEVSUwogICAgIyBDaGVjayBpZiBkZi5tZXRhIGlzIHZhbGlkLCBpZiBpdCBpcywgbG9vayBmb3IgYSBmZWF0dXJlIHZlY3RvcgogICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGZfYXJ0aWZhY3QuYXJ0aWZhY3RfdXJsKQogICAgaXNfZmVhdHVyZV92ZWN0b3IgPSBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXgKCiAgICAjIExvb2sgaW5zaWRlIG1ldGEuc3BlYy5sYWJlbF9mZWF0dXJlIHRvIGlkZW50aWZ5IHRoZSBsYWJlbF9jb2x1bW4gaWYgdGhlIHVzZXIgZGlkIG5vdCBzcGVjaWZ5IGl0CiAgICBpZiBsYWJlbF9jb2x1bW4gaXMgTm9uZToKICAgICAgICBpZiBpc19mZWF0dXJlX3ZlY3RvcjoKICAgICAgICAgICAgbGFiZWxfY29sdW1uID0gZGZfYXJ0aWZhY3QubWV0YS5zcGVjLmxhYmVsX2ZlYXR1cmUuc3BsaXQoIi4iKVsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIk5vIGxhYmVsX2NvbHVtbiB3YXMgZ2l2ZW4sIHBsZWFzZSBhZGQgYSBsYWJlbF9jb2x1bW4uIikKCiAgICAjIFVzZSB0aGUgZmVhdHVyZSB2ZWN0b3IgYXMgZGF0YWZyYW1lCiAgICBkZiA9IGRmX2FydGlmYWN0LmFzX2RmKCkKCiAgICAjIEVuc3VyZSBrIGlzIG5vdCBiaWdnZXIgdGhhbiB0aGUgdG90YWwgbnVtYmVyIG9mIGZlYXR1cmVzCiAgICBpZiBrID4gZGYuc2hhcGVbMV06CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJLIGNhbm5vdCBiZSBiaWdnZXIgdGhhbiB0aGUgdG90YWwgbnVtYmVyIG9mIGZlYXR1cmVzICh7ZGYuc2hhcGVbMV19KS4gUGxlYXNlIGNob29zZSBhIHNtYWxsZXIgSy4iCiAgICAgICAgKQogICAgZWxpZiBrIDwgMToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJLIGNhbm5vdCBiZSBzbWFsbGVyIHRoYW4gMS4gUGxlYXNlIGNob29zZSBhIGJpZ2dlciBLLiIpCgogICAgIyBDcmVhdGUgYSBzYW1wbGUgZGF0YWZyYW1lIG9mIHRoZSBvcmlnaW5hbCBmZWF0dXJlIHZlY3RvcgogICAgaWYgc2FtcGxlX3JhdGlvOgogICAgICAgIGRmID0gKAogICAgICAgICAgICBkZi5ncm91cGJ5KGxhYmVsX2NvbHVtbikKICAgICAgICAgICAgLmFwcGx5KGxhbWJkYSB4OiB4LnNhbXBsZShmcmFjPXNhbXBsZV9yYXRpbykpCiAgICAgICAgICAgIC5yZXNldF9pbmRleChkcm9wPVRydWUpCiAgICAgICAgKQogICAgICAgIGRmID0gZGYuZHJvcG5hKCkKCiAgICAjIFNldCBmZWF0dXJlIHZlY3RvciBhbmQgbGFiZWxzCiAgICB5ID0gZGYucG9wKGxhYmVsX2NvbHVtbikKICAgIFggPSBkZgoKICAgIGlmIG5wLm9iamVjdF8gaW4gbGlzdChYLmR0eXBlcykgYW5kIGlnbm9yZV90eXBlX2Vycm9ycyBpcyBGYWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIntkZi5zZWxlY3RfZHR5cGVzKGluY2x1ZGU9WydvYmplY3QnXSkuY29sdW1ucy50b2xpc3QoKX0gYXJlIG5laXRoZXIgZmxvYXQgb3IgaW50LiIKICAgICAgICApCgogICAgIyBDcmVhdGUgc2VsZWN0ZWQgc3RhdGlzdGljYWwgZXN0aW1hdG9ycwogICAgc3RhdF9mdW5jdGlvbnNfbGlzdCA9IHsKICAgICAgICBzdGF0X25hbWU6IFNlbGVjdEtCZXN0KAogICAgICAgICAgICBzY29yZV9mdW5jPWNyZWF0ZV9jbGFzcyhmInNrbGVhcm4uZmVhdHVyZV9zZWxlY3Rpb24ue3N0YXRfbmFtZX0iKSwgaz1rCiAgICAgICAgKQogICAgICAgIGZvciBzdGF0X25hbWUgaW4gc3RhdF9maWx0ZXJzCiAgICB9CiAgICByZXF1aXJlc19hYnMgPSBbImNoaTIiXQoKICAgICMgUnVuIHN0YXRpc3RpYyBmaWx0ZXJzCiAgICBzZWxlY3RlZF9mZWF0dXJlc19hZ2cgPSB7fQogICAgc3RhdHNfZGYgPSBwZC5EYXRhRnJhbWUoaW5kZXg9WC5jb2x1bW5zKS5kcm9wbmEoKQoKICAgIGZvciBzdGF0X25hbWUsIHN0YXRfZnVuYyBpbiBzdGF0X2Z1bmN0aW9uc19saXN0Lml0ZW1zKCk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICBwYXJhbXMgPSAoWCwgeSkgaWYgc3RhdF9uYW1lIGluIHJlcXVpcmVzX2FicyBlbHNlIChhYnMoWCksIHkpCiAgICAgICAgICAgIHN0YXQgPSBzdGF0X2Z1bmMuZml0KCpwYXJhbXMpCgogICAgICAgICAgICAjIENvbGxlY3Qgc3RhdCBmdW5jdGlvbiByZXN1bHRzCiAgICAgICAgICAgIHN0YXRfZGYgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgICAgICAgICBpbmRleD1YLmNvbHVtbnMsIGNvbHVtbnM9W3N0YXRfbmFtZV0sIGRhdGE9c3RhdC5zY29yZXNfCiAgICAgICAgICAgICkKICAgICAgICAgICAgcGxvdF9zdGF0KGNvbnRleHQsIHN0YXRfbmFtZSwgc3RhdF9kZikKICAgICAgICAgICAgc3RhdHNfZGYgPSBzdGF0c19kZi5qb2luKHN0YXRfZGYpCgogICAgICAgICAgICAjIFNlbGVjdCBLIEJlc3QgZmVhdHVyZXMKICAgICAgICAgICAgc2VsZWN0ZWRfZmVhdHVyZXMgPSBYLmNvbHVtbnNbc3RhdF9mdW5jLmdldF9zdXBwb3J0KCldCiAgICAgICAgICAgIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZ1tzdGF0X25hbWVdID0gc2VsZWN0ZWRfZmVhdHVyZXMKCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiQ291bGRuJ3QgY2FsY3VsYXRlIHtzdGF0X25hbWV9IGJlY2F1c2Ugb2Y6IHtlfSIpCgogICAgIyBDcmVhdGUgbW9kZWxzIGZyb20gY2xhc3MgbmFtZSAvIGpzb24gZmlsZSAvIGpzb24gcGFyYW1zCiAgICBhbGxfc2tsZWFybl9lc3RpbWF0b3JzID0gZGljdChhbGxfZXN0aW1hdG9ycygpKSBpZiBsZW4obW9kZWxfZmlsdGVycykgPiAwIGVsc2Uge30KICAgIHNlbGVjdGVkX21vZGVscyA9IHt9CiAgICBmb3IgbW9kZWxfbmFtZSwgbW9kZWwgaW4gbW9kZWxfZmlsdGVycy5pdGVtcygpOgogICAgICAgIGlmICIuanNvbiIgaW4gbW9kZWw6CiAgICAgICAgICAgIGN1cnJlbnRfbW9kZWwgPSBqc29uLmxvYWQob3Blbihtb2RlbCwgInIiKSkKICAgICAgICAgICAgY2xhc3NpZmllcl9jbGFzcyA9IGNyZWF0ZV9jbGFzcyhjdXJyZW50X21vZGVsWyJNRVRBIl1bImNsYXNzIl0pCiAgICAgICAgICAgIHNlbGVjdGVkX21vZGVsc1ttb2RlbF9uYW1lXSA9IGNsYXNzaWZpZXJfY2xhc3MoKipjdXJyZW50X21vZGVsWyJDTEFTUyJdKQogICAgICAgIGVsaWYgbW9kZWwgaW4gYWxsX3NrbGVhcm5fZXN0aW1hdG9yczoKICAgICAgICAgICAgc2VsZWN0ZWRfbW9kZWxzW21vZGVsX25hbWVdID0gYWxsX3NrbGVhcm5fZXN0aW1hdG9yc1ttb2RlbF9uYW1lXSgpCgogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGN1cnJlbnRfbW9kZWwgPSBqc29uLmxvYWRzKG1vZGVsKQogICAgICAgICAgICAgICAgY2xhc3NpZmllcl9jbGFzcyA9IGNyZWF0ZV9jbGFzcyhjdXJyZW50X21vZGVsWyJNRVRBIl1bImNsYXNzIl0pCiAgICAgICAgICAgICAgICBzZWxlY3RlZF9tb2RlbHNbbW9kZWxfbmFtZV0gPSBjbGFzc2lmaWVyX2NsYXNzKCoqY3VycmVudF9tb2RlbFsiQ0xBU1MiXSkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInVuYWJsZSB0byBsb2FkIHttb2RlbH0gYmVjYXVzZSBvZjoge2V9IikKCiAgICAjIFJ1biBtb2RlbCBmaWx0ZXJzCiAgICBtb2RlbHNfZGYgPSBwZC5EYXRhRnJhbWUoaW5kZXg9WC5jb2x1bW5zKQogICAgZm9yIG1vZGVsX25hbWUsIG1vZGVsIGluIHNlbGVjdGVkX21vZGVscy5pdGVtcygpOgoKICAgICAgICBpZiBtb2RlbF9uYW1lID09ICJMb2dpc3RpY1JlZ3Jlc3Npb24iOgogICAgICAgICAgICBtb2RlbC5zZXRfcGFyYW1zKHNvbHZlcj0ibGlibGluZWFyIikKCiAgICAgICAgIyBUcmFpbiBtb2RlbCBhbmQgZ2V0IGZlYXR1cmUgaW1wb3J0YW5jZQogICAgICAgIHNlbGVjdF9mcm9tX21vZGVsID0gU2VsZWN0RnJvbU1vZGVsKG1vZGVsKS5maXQoWCwgeSkKICAgICAgICBmZWF0dXJlX2lkeCA9IHNlbGVjdF9mcm9tX21vZGVsLmdldF9zdXBwb3J0KCkKICAgICAgICBmZWF0dXJlX25hbWVzID0gWC5jb2x1bW5zW2ZlYXR1cmVfaWR4XQogICAgICAgIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZ1ttb2RlbF9uYW1lXSA9IGZlYXR1cmVfbmFtZXMudG9saXN0KCkKCiAgICAgICAgIyBDb2xsZWN0IG1vZGVsIGZlYXR1cmUgaW1wb3J0YW5jZQogICAgICAgIGlmIGhhc2F0dHIoc2VsZWN0X2Zyb21fbW9kZWwuZXN0aW1hdG9yXywgImNvZWZfIik6CiAgICAgICAgICAgIHN0YXRfZGYgPSBzZWxlY3RfZnJvbV9tb2RlbC5lc3RpbWF0b3JfLmNvZWZfCiAgICAgICAgZWxpZiBoYXNhdHRyKHNlbGVjdF9mcm9tX21vZGVsLmVzdGltYXRvcl8sICJmZWF0dXJlX2ltcG9ydGFuY2VzXyIpOgogICAgICAgICAgICBzdGF0X2RmID0gc2VsZWN0X2Zyb21fbW9kZWwuZXN0aW1hdG9yXy5mZWF0dXJlX2ltcG9ydGFuY2VzXwoKICAgICAgICBzdGF0X2RmID0gcGQuRGF0YUZyYW1lKGluZGV4PVguY29sdW1ucywgY29sdW1ucz1bbW9kZWxfbmFtZV0sIGRhdGE9c3RhdF9kZlswXSkKICAgICAgICBtb2RlbHNfZGYgPSBtb2RlbHNfZGYuam9pbihzdGF0X2RmKQoKICAgICAgICBwbG90X3N0YXQoY29udGV4dCwgbW9kZWxfbmFtZSwgc3RhdF9kZikKCiAgICAjIENyZWF0ZSBmZWF0dXJlX3Njb3JlcyBERiB3aXRoIHN0YXQgJiBtb2RlbCBmaWx0ZXJzIHNjb3JlcwogICAgcmVzdWx0X21hdHJpeF9kZiA9IHBkLmNvbmNhdChbc3RhdHNfZGYsIG1vZGVsc19kZl0sIGF4aXM9MSwgc29ydD1GYWxzZSkKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoCiAgICAgICAga2V5PSJmZWF0dXJlX3Njb3JlcyIsCiAgICAgICAgZGY9cmVzdWx0X21hdHJpeF9kZiwKICAgICAgICBsb2NhbF9wYXRoPSJmZWF0dXJlX3Njb3Jlcy5wYXJxdWV0IiwKICAgICAgICBmb3JtYXQ9InBhcnF1ZXQiLAogICAgKQogICAgaWYgbWF4X3NjYWxlZF9zY29yZXM6CiAgICAgICAgbm9ybWFsaXplZF9kZiA9IHJlc3VsdF9tYXRyaXhfZGYucmVwbGFjZShbbnAuaW5mLCAtbnAuaW5mXSwgbnAubmFuKS52YWx1ZXMKICAgICAgICBtaW5fbWF4X3NjYWxlciA9IE1pbk1heFNjYWxlcigpCiAgICAgICAgbm9ybWFsaXplZF9kZiA9IG1pbl9tYXhfc2NhbGVyLmZpdF90cmFuc2Zvcm0obm9ybWFsaXplZF9kZikKICAgICAgICBub3JtYWxpemVkX2RmID0gcGQuRGF0YUZyYW1lKAogICAgICAgICAgICBkYXRhPW5vcm1hbGl6ZWRfZGYsCiAgICAgICAgICAgIGNvbHVtbnM9cmVzdWx0X21hdHJpeF9kZi5jb2x1bW5zLAogICAgICAgICAgICBpbmRleD1yZXN1bHRfbWF0cml4X2RmLmluZGV4LAogICAgICAgICkKICAgICAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgICAgICBrZXk9Im1heF9zY2FsZWRfc2NvcmVzX2ZlYXR1cmVfc2NvcmVzIiwKICAgICAgICAgICAgZGY9bm9ybWFsaXplZF9kZiwKICAgICAgICAgICAgbG9jYWxfcGF0aD0ibWF4X3NjYWxlZF9zY29yZXNfZmVhdHVyZV9zY29yZXMucGFycXVldCIsCiAgICAgICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICAgICAgKQoKICAgICMgQ3JlYXRlIGZlYXR1cmUgY291bnQgRGF0YUZyYW1lCiAgICBmb3IgdGVzdF9uYW1lIGluIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZzoKICAgICAgICByZXN1bHRfbWF0cml4X2RmW3Rlc3RfbmFtZV0gPSBbCiAgICAgICAgICAgIDEgaWYgeCBpbiBzZWxlY3RlZF9mZWF0dXJlc19hZ2dbdGVzdF9uYW1lXSBlbHNlIDAgZm9yIHggaW4gWC5jb2x1bW5zCiAgICAgICAgXQogICAgcmVzdWx0X21hdHJpeF9kZi5sb2NbOiwgIm51bV92b3RlcyJdID0gcmVzdWx0X21hdHJpeF9kZi5zdW0oYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICBrZXk9InNlbGVjdGVkX2ZlYXR1cmVzX2NvdW50IiwKICAgICAgICBkZj1yZXN1bHRfbWF0cml4X2RmLAogICAgICAgIGxvY2FsX3BhdGg9InNlbGVjdGVkX2ZlYXR1cmVzX2NvdW50LnBhcnF1ZXQiLAogICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICApCgogICAgIyBIb3cgbWFueSB2b3RlcyBhcmUgbmVlZGVkIGZvciBhIGZlYXR1cmUgdG8gYmUgc2VsZWN0ZWQ/CiAgICBpZiBpc2luc3RhbmNlKG1pbl92b3RlcywgaW50KToKICAgICAgICB2b3Rlc19uZWVkZWQgPSBtaW5fdm90ZXMKICAgIGVsc2U6CiAgICAgICAgbnVtX2ZpbHRlcnMgPSBsZW4oc3RhdF9maWx0ZXJzKSArIGxlbihtb2RlbF9maWx0ZXJzKQogICAgICAgIHZvdGVzX25lZWRlZCA9IGludChucC5mbG9vcihudW1fZmlsdGVycyAqIG1heChtaW4obWluX3ZvdGVzLCAxKSwgMCkpKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInZvdGVzIG5lZWRlZCB0byBiZSBzZWxlY3RlZDoge3ZvdGVzX25lZWRlZH0iKQoKICAgICMgQ3JlYXRlIGZpbmFsIGZlYXR1cmUgZGF0YWZyYW1lCiAgICBzZWxlY3RlZF9mZWF0dXJlcyA9IHJlc3VsdF9tYXRyaXhfZGZbCiAgICAgICAgcmVzdWx0X21hdHJpeF9kZi5udW1fdm90ZXMgPj0gdm90ZXNfbmVlZGVkCiAgICBdLmluZGV4LnRvbGlzdCgpCiAgICBnb29kX2ZlYXR1cmVfZGYgPSBkZi5sb2NbOiwgc2VsZWN0ZWRfZmVhdHVyZXNdCiAgICBmaW5hbF9kZiA9IHBkLmNvbmNhdChbZ29vZF9mZWF0dXJlX2RmLCB5XSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICBrZXk9InNlbGVjdGVkX2ZlYXR1cmVzIiwKICAgICAgICBkZj1maW5hbF9kZiwKICAgICAgICBsb2NhbF9wYXRoPSJzZWxlY3RlZF9mZWF0dXJlcy5wYXJxdWV0IiwKICAgICAgICBmb3JtYXQ9InBhcnF1ZXQiLAogICAgKQoKICAgICMgQ3JlYXRpbmcgYSBuZXcgZmVhdHVyZSB2ZWN0b3IgY29udGFpbmluZyBvbmx5IHRoZSBpZGVudGlmaWVkIHRvcCBmZWF0dXJlcwogICAgaWYgaXNfZmVhdHVyZV92ZWN0b3IgYW5kIGRmX2FydGlmYWN0Lm1ldGEuc3BlYy5mZWF0dXJlcyBhbmQgb3V0cHV0X3ZlY3Rvcl9uYW1lOgogICAgICAgICMgU2VsZWN0aW5nIHRoZSB0b3AgSyBmZWF0dXJlcyBmcm9tIG91ciB0b3AgZmVhdHVyZSBkYXRhZnJhbWUKICAgICAgICBzZWxlY3RlZF9mZWF0dXJlcyA9IHJlc3VsdF9tYXRyaXhfZGYuaGVhZChrKS5pbmRleAoKICAgICAgICAjIE1hdGNoIHRoZSBzZWxlY3RlZCBmZWF0dXJlIG5hbWVzIHRvIHRoZSBGUyBGZWF0dXJlIGFubm90YXRpb25zCiAgICAgICAgbWF0Y2hlZF9zZWxlY3Rpb25zID0gWwogICAgICAgICAgICBmZWF0dXJlCiAgICAgICAgICAgIGZvciBmZWF0dXJlIGluIGxpc3QoZGZfYXJ0aWZhY3QubWV0YS5zcGVjLmZlYXR1cmVzKQogICAgICAgICAgICBmb3Igc2VsZWN0ZWQgaW4gbGlzdChzZWxlY3RlZF9mZWF0dXJlcykKICAgICAgICAgICAgaWYgZmVhdHVyZS5lbmRzd2l0aChzZWxlY3RlZCkKICAgICAgICBdCgogICAgICAgICMgRGVmaW5pbmcgb3VyIG5ldyBmZWF0dXJlIHZlY3RvcgogICAgICAgIHRvcF9mZWF0dXJlc19mdiA9IGZzLkZlYXR1cmVWZWN0b3IoCiAgICAgICAgICAgIG91dHB1dF92ZWN0b3JfbmFtZSwKICAgICAgICAgICAgbWF0Y2hlZF9zZWxlY3Rpb25zLAogICAgICAgICAgICBsYWJlbF9mZWF0dXJlPSJsYWJlbHMubGFiZWwiLAogICAgICAgICAgICBkZXNjcmlwdGlvbj0iZmVhdHVyZSB2ZWN0b3IgY29tcG9zZWQgc3RyaWN0bHkgb2Ygb3VyIHRvcCBmZWF0dXJlcyIsCiAgICAgICAgKQoKICAgICAgICAjIFNhdmluZwogICAgICAgIHRvcF9mZWF0dXJlc19mdi5zYXZlKCkKICAgICAgICBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcyh0b3BfZmVhdHVyZXNfZnYsIHRhcmdldD1QYXJxdWV0VGFyZ2V0KCkpCgogICAgICAgICMgTG9nZ2luZyBvdXIgbmV3IGZlYXR1cmUgdmVjdG9yIFVSSQogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdCgidG9wX2ZlYXR1cmVzX3ZlY3RvciIsIHRvcF9mZWF0dXJlc19mdi51cmkpCg== + code_origin: '' default_handler: feature_selection - disable_auto_mount: false - clone_target_dir: '' - env: [] - priority_class_name: '' - preemption_mode: prevent - affinity: null - tolerations: null - security_context: {} -verbose: false + description: Select features through multiple Statistical and Model filters + image: mlrun/mlrun +metadata: + categories: + - data-preparation + - machine-learning + tag: '' + name: feature-selection diff --git a/feature_selection/item.yaml b/feature_selection/item.yaml index 7e80a417b..1c79e6f12 100644 --- a/feature_selection/item.yaml +++ b/feature_selection/item.yaml @@ -12,9 +12,9 @@ labels: author: orz maintainers: [] marketplaceType: '' -mlrunVersion: 1.1.0 +mlrunVersion: 1.7.0 name: feature-selection -platformVersion: 3.5.0 +platformVersion: 3.6.0 spec: filename: feature_selection.py handler: feature_selection @@ -22,4 +22,4 @@ spec: kind: job requirements: [] url: '' -version: 1.4.0 +version: 1.5.0 From 64907af0011894ceffe6e7e8471a306eb215221f Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Mon, 2 Sep 2024 14:52:10 +0300 Subject: [PATCH 3/8] [feature selection] update function yaml --- feature_selection/function.yaml | 56 ++++++++++++++++----------------- feature_selection/item.yaml | 2 +- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/feature_selection/function.yaml b/feature_selection/function.yaml index d4e95f1e9..f1bf53b8a 100644 --- a/feature_selection/function.yaml +++ b/feature_selection/function.yaml @@ -1,33 +1,43 @@ +metadata: + name: feature-selection + tag: '' + categories: + - data-preparation + - machine-learning kind: job -verbose: false spec: - disable_auto_mount: false - command: '' entry_points: show_values_on_bars: - lineno: 54 + doc: '' + has_kwargs: false parameters: - name: axs - name: h_v default: v - name: space default: 0.4 - name: show_values_on_bars + lineno: 54 has_varargs: false - has_kwargs: false - doc: '' + name: show_values_on_bars plot_stat: - lineno: 76 + doc: '' + has_kwargs: false parameters: - name: context - name: stat_name - name: stat_df - name: plot_stat + lineno: 76 has_varargs: false - has_kwargs: false - doc: '' + name: plot_stat feature_selection: - lineno: 106 + doc: 'Applies selected feature selection statistical functions or models on + our ''df_artifact''. + + + Each statistical function or model will vote for it''s best K selected features. + + If a feature has >= ''min_votes'' votes, it will be selected.' + has_kwargs: false parameters: - name: context doc: the function context. @@ -78,26 +88,16 @@ spec: type: bool doc: bool stating if the data is passed as a feature vector. default: false - name: feature_selection + lineno: 106 has_varargs: false - has_kwargs: false - doc: 'Applies selected feature selection statistical functions or models on - our ''df_artifact''. - - - Each statistical function or model will vote for it''s best K selected features. - - If a feature has >= ''min_votes'' votes, it will be selected.' + name: feature_selection + disable_auto_mount: false + command: '' build: origin_filename: '' functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IGpzb24KaW1wb3J0IG9zCgppbXBvcnQgbWF0cGxvdGxpYi5weXBsb3QgYXMgcGx0CmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgbWxydW4uZmVhdHVyZV9zdG9yZSBhcyBmcwppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgc2VhYm9ybiBhcyBzbnMKZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IFBsb3RBcnRpZmFjdApmcm9tIG1scnVuLmRhdGFzdG9yZS50YXJnZXRzIGltcG9ydCBQYXJxdWV0VGFyZ2V0CiMgTUxSdW4gdXRpbHMKZnJvbSBtbHJ1bi51dGlscy5oZWxwZXJzIGltcG9ydCBjcmVhdGVfY2xhc3MKIyBGZWF0dXJlIHNlbGVjdGlvbiBzdHJhdGVnaWVzCmZyb20gc2tsZWFybi5mZWF0dXJlX3NlbGVjdGlvbiBpbXBvcnQgU2VsZWN0RnJvbU1vZGVsLCBTZWxlY3RLQmVzdAojIFNjYWxlIGZlYXR1cmUgc2NvcmVzZ2l0IHN0CmZyb20gc2tsZWFybi5wcmVwcm9jZXNzaW5nIGltcG9ydCBNaW5NYXhTY2FsZXIKIyBTS0xlYXJuIGVzdGltYXRvcnMgbGlzdApmcm9tIHNrbGVhcm4udXRpbHMgaW1wb3J0IGFsbF9lc3RpbWF0b3JzCgpERUZBVUxUX1NUQVRfRklMVEVSUyA9IFsiZl9jbGFzc2lmIiwgIm11dHVhbF9pbmZvX2NsYXNzaWYiLCAiY2hpMiIsICJmX3JlZ3Jlc3Npb24iXQpERUZBVUxUX01PREVMX0ZJTFRFUlMgPSB7CiAgICAiTGluZWFyU1ZDIjogIkxpbmVhclNWQyIsCiAgICAiTG9naXN0aWNSZWdyZXNzaW9uIjogIkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAiRXh0cmFUcmVlc0NsYXNzaWZpZXIiOiAiRXh0cmFUcmVlc0NsYXNzaWZpZXIiLAp9CgoKZGVmIF9jbGVhcl9jdXJyZW50X2ZpZ3VyZSgpOgogICAgIiIiCiAgICBDbGVhciBtYXRwbG90bGliIGN1cnJlbnQgZmlndXJlLgogICAgIiIiCiAgICBwbHQuY2xhKCkKICAgIHBsdC5jbGYoKQogICAgcGx0LmNsb3NlKCkKCgpkZWYgc2hvd192YWx1ZXNfb25fYmFycyhheHMsIGhfdj0idiIsIHNwYWNlPTAuNCk6CiAgICBkZWYgX3Nob3dfb25fc2luZ2xlX3Bsb3QoYXhfKToKICAgICAgICBpZiBoX3YgPT0gInYiOgogICAgICAgICAgICBmb3IgcCBpbiBheF8ucGF0Y2hlczoKICAgICAgICAgICAgICAgIF94ID0gcC5nZXRfeCgpICsgcC5nZXRfd2lkdGgoKSAvIDIKICAgICAgICAgICAgICAgIF95ID0gcC5nZXRfeSgpICsgcC5nZXRfaGVpZ2h0KCkKICAgICAgICAgICAgICAgIHZhbHVlID0gaW50KHAuZ2V0X2hlaWdodCgpKQogICAgICAgICAgICAgICAgYXhfLnRleHQoX3gsIF95LCB2YWx1ZSwgaGE9ImNlbnRlciIpCiAgICAgICAgZWxpZiBoX3YgPT0gImgiOgogICAgICAgICAgICBmb3IgcCBpbiBheF8ucGF0Y2hlczoKICAgICAgICAgICAgICAgIF94ID0gcC5nZXRfeCgpICsgcC5nZXRfd2lkdGgoKSArIGZsb2F0KHNwYWNlKQogICAgICAgICAgICAgICAgX3kgPSBwLmdldF95KCkgKyBwLmdldF9oZWlnaHQoKQogICAgICAgICAgICAgICAgdmFsdWUgPSBpbnQocC5nZXRfd2lkdGgoKSkKICAgICAgICAgICAgICAgIGF4Xy50ZXh0KF94LCBfeSwgdmFsdWUsIGhhPSJsZWZ0IikKCiAgICBpZiBpc2luc3RhbmNlKGF4cywgbnAubmRhcnJheSk6CiAgICAgICAgZm9yIGlkeCwgYXggaW4gbnAubmRlbnVtZXJhdGUoYXhzKToKICAgICAgICAgICAgX3Nob3dfb25fc2luZ2xlX3Bsb3QoYXgpCiAgICBlbHNlOgogICAgICAgIF9zaG93X29uX3NpbmdsZV9wbG90KGF4cykKCgpkZWYgcGxvdF9zdGF0KGNvbnRleHQsIHN0YXRfbmFtZSwgc3RhdF9kZik6CiAgICBfY2xlYXJfY3VycmVudF9maWd1cmUoKQoKICAgICMgQWRkIGNoYXJ0CiAgICBheCA9IHBsdC5heGVzKCkKICAgIHN0YXRfY2hhcnQgPSBzbnMuYmFycGxvdCgKICAgICAgICB4PXN0YXRfbmFtZSwKICAgICAgICB5PSJpbmRleCIsCiAgICAgICAgZGF0YT1zdGF0X2RmLnNvcnRfdmFsdWVzKHN0YXRfbmFtZSwgYXNjZW5kaW5nPUZhbHNlKS5yZXNldF9pbmRleCgpLAogICAgICAgIGF4PWF4LAogICAgKQogICAgcGx0LnRpZ2h0X2xheW91dCgpCgogICAgZm9yIHAgaW4gc3RhdF9jaGFydC5wYXRjaGVzOgogICAgICAgIHdpZHRoID0gcC5nZXRfd2lkdGgoKQogICAgICAgIHBsdC50ZXh0KAogICAgICAgICAgICA1ICsgcC5nZXRfd2lkdGgoKSwKICAgICAgICAgICAgcC5nZXRfeSgpICsgMC41NSAqIHAuZ2V0X2hlaWdodCgpLAogICAgICAgICAgICAiezoxLjJmfSIuZm9ybWF0KHdpZHRoKSwKICAgICAgICAgICAgaGE9ImNlbnRlciIsCiAgICAgICAgICAgIHZhPSJjZW50ZXIiLAogICAgICAgICkKCiAgICBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBQbG90QXJ0aWZhY3QoZiJ7c3RhdF9uYW1lfSIsIGJvZHk9cGx0LmdjZigpKSwKICAgICAgICBsb2NhbF9wYXRoPW9zLnBhdGguam9pbigicGxvdHMiLCAiZmVhdHVyZV9zZWxlY3Rpb24iLCBmIntzdGF0X25hbWV9Lmh0bWwiKSwKICAgICkKICAgIF9jbGVhcl9jdXJyZW50X2ZpZ3VyZSgpCgoKZGVmIGZlYXR1cmVfc2VsZWN0aW9uKAogICAgY29udGV4dCwKICAgIGRmX2FydGlmYWN0LAogICAgazogaW50ID0gNSwKICAgIG1pbl92b3RlczogZmxvYXQgPSAwLjUsCiAgICBsYWJlbF9jb2x1bW46IHN0ciA9IE5vbmUsCiAgICBzdGF0X2ZpbHRlcnM6IGxpc3QgPSBOb25lLAogICAgbW9kZWxfZmlsdGVyczogZGljdCA9IE5vbmUsCiAgICBtYXhfc2NhbGVkX3Njb3JlczogYm9vbCA9IFRydWUsCiAgICBzYW1wbGVfcmF0aW86IGZsb2F0ID0gTm9uZSwKICAgIG91dHB1dF92ZWN0b3JfbmFtZTogZmxvYXQgPSBOb25lLAogICAgaWdub3JlX3R5cGVfZXJyb3JzOiBib29sID0gRmFsc2UsCiAgICBpc19mZWF0dXJlX3ZlY3RvcjogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBBcHBsaWVzIHNlbGVjdGVkIGZlYXR1cmUgc2VsZWN0aW9uIHN0YXRpc3RpY2FsIGZ1bmN0aW9ucyBvciBtb2RlbHMgb24gb3VyICdkZl9hcnRpZmFjdCcuCgogICAgRWFjaCBzdGF0aXN0aWNhbCBmdW5jdGlvbiBvciBtb2RlbCB3aWxsIHZvdGUgZm9yIGl0J3MgYmVzdCBLIHNlbGVjdGVkIGZlYXR1cmVzLgogICAgSWYgYSBmZWF0dXJlIGhhcyA+PSAnbWluX3ZvdGVzJyB2b3RlcywgaXQgd2lsbCBiZSBzZWxlY3RlZC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgdGhlIGZ1bmN0aW9uIGNvbnRleHQuCiAgICA6cGFyYW0gZGZfYXJ0aWZhY3Q6ICAgICAgICAgZGF0YWZyYW1lIHRvIHBhc3MgYXMgaW5wdXQuCiAgICA6cGFyYW0gazogICAgICAgICAgICAgICAgICAgbnVtYmVyIG9mIHRvcCBmZWF0dXJlcyB0byBzZWxlY3QgZnJvbSBlYWNoIHN0YXRpc3RpY2FsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnVuY3Rpb24gb3IgbW9kZWwuCiAgICA6cGFyYW0gbWluX3ZvdGVzOiAgICAgICAgICAgbWluaW1hbCBudW1iZXIgb2Ygdm90ZXMgKGZyb20gYSBtb2RlbCBvciBieSBzdGF0aXN0aWNhbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZ1bmN0aW9uKSBuZWVkZWQgZm9yIGEgZmVhdHVyZSB0byBiZSBzZWxlY3RlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgc3BlY2lmaWVkIGJ5IHBlcmNlbnRhZ2Ugb2Ygdm90ZXMgb3IgYWJzb2x1dGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBudW1iZXIgb2Ygdm90ZXMuCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uOiAgICAgICAgZ3JvdW5kLXRydXRoICh5KSBsYWJlbHMuCiAgICA6cGFyYW0gc3RhdF9maWx0ZXJzOiAgICAgICAgc3RhdGlzdGljYWwgZnVuY3Rpb25zIHRvIGFwcGx5IHRvIHRoZSBmZWF0dXJlcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChmcm9tIHNrbGVhcm4uZmVhdHVyZV9zZWxlY3Rpb24pLgogICAgOnBhcmFtIG1vZGVsX2ZpbHRlcnM6ICAgICAgIG1vZGVscyB0byB1c2UgZm9yIGZlYXR1cmUgZXZhbHVhdGlvbiwgY2FuIGJlIHNwZWNpZmllZCBieQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsIG5hbWUgKGV4LiBMaW5lYXJTVkMpLCBmb3JtYWxpemVkIGpzb24gKGNvbnRhaW5zICdDTEFTUycsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ0ZJVCcsICdNRVRBJykgb3IgYSBwYXRoIHRvIHN1Y2gganNvbiBmaWxlLgogICAgOnBhcmFtIG1heF9zY2FsZWRfc2NvcmVzOiAgIHByb2R1Y2UgZmVhdHVyZSBzY29yZXMgdGFibGUgc2NhbGVkIHdpdGggbWF4X3NjYWxlci4KICAgIDpwYXJhbSBzYW1wbGVfcmF0aW86ICAgICAgICBwZXJjZW50YWdlIG9mIHRoZSBkYXRhc2V0IHRoZSB1c2VyIHdoaXNoZXMgdG8gY29tcHV0ZSB0aGUgZmVhdHVyZSBzZWxlY3Rpb24gcHJvY2VzcyBvbi4KICAgIDpwYXJhbSBvdXRwdXRfdmVjdG9yX25hbWU6ICBjcmVhdGVzIGEgbmV3IGZlYXR1cmUgdmVjdG9yIGNvbnRhaW5pbmcgb25seSB0aGUgaWRlbnRpZmllcyBmZWF0dXJlcy4KICAgIDpwYXJhbSBpZ25vcmVfdHlwZV9lcnJvcnM6ICBza2lwcyBkYXRhdHlwZXMgdGhhdCBhcmUgbmVpdGhlciBmbG9hdCBub3IgaW50IHdpdGhpbiB0aGUgZmVhdHVyZSB2ZWN0b3IuCiAgICA6cGFyYW0gaXNfZmVhdHVyZV92ZWN0b3I6ICAgYm9vbCBzdGF0aW5nIGlmIHRoZSBkYXRhIGlzIHBhc3NlZCBhcyBhIGZlYXR1cmUgdmVjdG9yLgogICAgIiIiCiAgICBzdGF0X2ZpbHRlcnMgPSBzdGF0X2ZpbHRlcnMgb3IgREVGQVVMVF9TVEFUX0ZJTFRFUlMKICAgIG1vZGVsX2ZpbHRlcnMgPSBtb2RlbF9maWx0ZXJzIG9yIERFRkFVTFRfTU9ERUxfRklMVEVSUwogICAgIyBDaGVjayBpZiBkZi5tZXRhIGlzIHZhbGlkLCBpZiBpdCBpcywgbG9vayBmb3IgYSBmZWF0dXJlIHZlY3RvcgogICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGZfYXJ0aWZhY3QuYXJ0aWZhY3RfdXJsKQogICAgaXNfZmVhdHVyZV92ZWN0b3IgPSBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXgKCiAgICAjIExvb2sgaW5zaWRlIG1ldGEuc3BlYy5sYWJlbF9mZWF0dXJlIHRvIGlkZW50aWZ5IHRoZSBsYWJlbF9jb2x1bW4gaWYgdGhlIHVzZXIgZGlkIG5vdCBzcGVjaWZ5IGl0CiAgICBpZiBsYWJlbF9jb2x1bW4gaXMgTm9uZToKICAgICAgICBpZiBpc19mZWF0dXJlX3ZlY3RvcjoKICAgICAgICAgICAgbGFiZWxfY29sdW1uID0gZGZfYXJ0aWZhY3QubWV0YS5zcGVjLmxhYmVsX2ZlYXR1cmUuc3BsaXQoIi4iKVsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIk5vIGxhYmVsX2NvbHVtbiB3YXMgZ2l2ZW4sIHBsZWFzZSBhZGQgYSBsYWJlbF9jb2x1bW4uIikKCiAgICAjIFVzZSB0aGUgZmVhdHVyZSB2ZWN0b3IgYXMgZGF0YWZyYW1lCiAgICBkZiA9IGRmX2FydGlmYWN0LmFzX2RmKCkKCiAgICAjIEVuc3VyZSBrIGlzIG5vdCBiaWdnZXIgdGhhbiB0aGUgdG90YWwgbnVtYmVyIG9mIGZlYXR1cmVzCiAgICBpZiBrID4gZGYuc2hhcGVbMV06CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJLIGNhbm5vdCBiZSBiaWdnZXIgdGhhbiB0aGUgdG90YWwgbnVtYmVyIG9mIGZlYXR1cmVzICh7ZGYuc2hhcGVbMV19KS4gUGxlYXNlIGNob29zZSBhIHNtYWxsZXIgSy4iCiAgICAgICAgKQogICAgZWxpZiBrIDwgMToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJLIGNhbm5vdCBiZSBzbWFsbGVyIHRoYW4gMS4gUGxlYXNlIGNob29zZSBhIGJpZ2dlciBLLiIpCgogICAgIyBDcmVhdGUgYSBzYW1wbGUgZGF0YWZyYW1lIG9mIHRoZSBvcmlnaW5hbCBmZWF0dXJlIHZlY3RvcgogICAgaWYgc2FtcGxlX3JhdGlvOgogICAgICAgIGRmID0gKAogICAgICAgICAgICBkZi5ncm91cGJ5KGxhYmVsX2NvbHVtbikKICAgICAgICAgICAgLmFwcGx5KGxhbWJkYSB4OiB4LnNhbXBsZShmcmFjPXNhbXBsZV9yYXRpbykpCiAgICAgICAgICAgIC5yZXNldF9pbmRleChkcm9wPVRydWUpCiAgICAgICAgKQogICAgICAgIGRmID0gZGYuZHJvcG5hKCkKCiAgICAjIFNldCBmZWF0dXJlIHZlY3RvciBhbmQgbGFiZWxzCiAgICB5ID0gZGYucG9wKGxhYmVsX2NvbHVtbikKICAgIFggPSBkZgoKICAgIGlmIG5wLm9iamVjdF8gaW4gbGlzdChYLmR0eXBlcykgYW5kIGlnbm9yZV90eXBlX2Vycm9ycyBpcyBGYWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIntkZi5zZWxlY3RfZHR5cGVzKGluY2x1ZGU9WydvYmplY3QnXSkuY29sdW1ucy50b2xpc3QoKX0gYXJlIG5laXRoZXIgZmxvYXQgb3IgaW50LiIKICAgICAgICApCgogICAgIyBDcmVhdGUgc2VsZWN0ZWQgc3RhdGlzdGljYWwgZXN0aW1hdG9ycwogICAgc3RhdF9mdW5jdGlvbnNfbGlzdCA9IHsKICAgICAgICBzdGF0X25hbWU6IFNlbGVjdEtCZXN0KAogICAgICAgICAgICBzY29yZV9mdW5jPWNyZWF0ZV9jbGFzcyhmInNrbGVhcm4uZmVhdHVyZV9zZWxlY3Rpb24ue3N0YXRfbmFtZX0iKSwgaz1rCiAgICAgICAgKQogICAgICAgIGZvciBzdGF0X25hbWUgaW4gc3RhdF9maWx0ZXJzCiAgICB9CiAgICByZXF1aXJlc19hYnMgPSBbImNoaTIiXQoKICAgICMgUnVuIHN0YXRpc3RpYyBmaWx0ZXJzCiAgICBzZWxlY3RlZF9mZWF0dXJlc19hZ2cgPSB7fQogICAgc3RhdHNfZGYgPSBwZC5EYXRhRnJhbWUoaW5kZXg9WC5jb2x1bW5zKS5kcm9wbmEoKQoKICAgIGZvciBzdGF0X25hbWUsIHN0YXRfZnVuYyBpbiBzdGF0X2Z1bmN0aW9uc19saXN0Lml0ZW1zKCk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICBwYXJhbXMgPSAoWCwgeSkgaWYgc3RhdF9uYW1lIGluIHJlcXVpcmVzX2FicyBlbHNlIChhYnMoWCksIHkpCiAgICAgICAgICAgIHN0YXQgPSBzdGF0X2Z1bmMuZml0KCpwYXJhbXMpCgogICAgICAgICAgICAjIENvbGxlY3Qgc3RhdCBmdW5jdGlvbiByZXN1bHRzCiAgICAgICAgICAgIHN0YXRfZGYgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgICAgICAgICBpbmRleD1YLmNvbHVtbnMsIGNvbHVtbnM9W3N0YXRfbmFtZV0sIGRhdGE9c3RhdC5zY29yZXNfCiAgICAgICAgICAgICkKICAgICAgICAgICAgcGxvdF9zdGF0KGNvbnRleHQsIHN0YXRfbmFtZSwgc3RhdF9kZikKICAgICAgICAgICAgc3RhdHNfZGYgPSBzdGF0c19kZi5qb2luKHN0YXRfZGYpCgogICAgICAgICAgICAjIFNlbGVjdCBLIEJlc3QgZmVhdHVyZXMKICAgICAgICAgICAgc2VsZWN0ZWRfZmVhdHVyZXMgPSBYLmNvbHVtbnNbc3RhdF9mdW5jLmdldF9zdXBwb3J0KCldCiAgICAgICAgICAgIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZ1tzdGF0X25hbWVdID0gc2VsZWN0ZWRfZmVhdHVyZXMKCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiQ291bGRuJ3QgY2FsY3VsYXRlIHtzdGF0X25hbWV9IGJlY2F1c2Ugb2Y6IHtlfSIpCgogICAgIyBDcmVhdGUgbW9kZWxzIGZyb20gY2xhc3MgbmFtZSAvIGpzb24gZmlsZSAvIGpzb24gcGFyYW1zCiAgICBhbGxfc2tsZWFybl9lc3RpbWF0b3JzID0gZGljdChhbGxfZXN0aW1hdG9ycygpKSBpZiBsZW4obW9kZWxfZmlsdGVycykgPiAwIGVsc2Uge30KICAgIHNlbGVjdGVkX21vZGVscyA9IHt9CiAgICBmb3IgbW9kZWxfbmFtZSwgbW9kZWwgaW4gbW9kZWxfZmlsdGVycy5pdGVtcygpOgogICAgICAgIGlmICIuanNvbiIgaW4gbW9kZWw6CiAgICAgICAgICAgIGN1cnJlbnRfbW9kZWwgPSBqc29uLmxvYWQob3Blbihtb2RlbCwgInIiKSkKICAgICAgICAgICAgY2xhc3NpZmllcl9jbGFzcyA9IGNyZWF0ZV9jbGFzcyhjdXJyZW50X21vZGVsWyJNRVRBIl1bImNsYXNzIl0pCiAgICAgICAgICAgIHNlbGVjdGVkX21vZGVsc1ttb2RlbF9uYW1lXSA9IGNsYXNzaWZpZXJfY2xhc3MoKipjdXJyZW50X21vZGVsWyJDTEFTUyJdKQogICAgICAgIGVsaWYgbW9kZWwgaW4gYWxsX3NrbGVhcm5fZXN0aW1hdG9yczoKICAgICAgICAgICAgc2VsZWN0ZWRfbW9kZWxzW21vZGVsX25hbWVdID0gYWxsX3NrbGVhcm5fZXN0aW1hdG9yc1ttb2RlbF9uYW1lXSgpCgogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGN1cnJlbnRfbW9kZWwgPSBqc29uLmxvYWRzKG1vZGVsKQogICAgICAgICAgICAgICAgY2xhc3NpZmllcl9jbGFzcyA9IGNyZWF0ZV9jbGFzcyhjdXJyZW50X21vZGVsWyJNRVRBIl1bImNsYXNzIl0pCiAgICAgICAgICAgICAgICBzZWxlY3RlZF9tb2RlbHNbbW9kZWxfbmFtZV0gPSBjbGFzc2lmaWVyX2NsYXNzKCoqY3VycmVudF9tb2RlbFsiQ0xBU1MiXSkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInVuYWJsZSB0byBsb2FkIHttb2RlbH0gYmVjYXVzZSBvZjoge2V9IikKCiAgICAjIFJ1biBtb2RlbCBmaWx0ZXJzCiAgICBtb2RlbHNfZGYgPSBwZC5EYXRhRnJhbWUoaW5kZXg9WC5jb2x1bW5zKQogICAgZm9yIG1vZGVsX25hbWUsIG1vZGVsIGluIHNlbGVjdGVkX21vZGVscy5pdGVtcygpOgoKICAgICAgICBpZiBtb2RlbF9uYW1lID09ICJMb2dpc3RpY1JlZ3Jlc3Npb24iOgogICAgICAgICAgICBtb2RlbC5zZXRfcGFyYW1zKHNvbHZlcj0ibGlibGluZWFyIikKCiAgICAgICAgIyBUcmFpbiBtb2RlbCBhbmQgZ2V0IGZlYXR1cmUgaW1wb3J0YW5jZQogICAgICAgIHNlbGVjdF9mcm9tX21vZGVsID0gU2VsZWN0RnJvbU1vZGVsKG1vZGVsKS5maXQoWCwgeSkKICAgICAgICBmZWF0dXJlX2lkeCA9IHNlbGVjdF9mcm9tX21vZGVsLmdldF9zdXBwb3J0KCkKICAgICAgICBmZWF0dXJlX25hbWVzID0gWC5jb2x1bW5zW2ZlYXR1cmVfaWR4XQogICAgICAgIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZ1ttb2RlbF9uYW1lXSA9IGZlYXR1cmVfbmFtZXMudG9saXN0KCkKCiAgICAgICAgIyBDb2xsZWN0IG1vZGVsIGZlYXR1cmUgaW1wb3J0YW5jZQogICAgICAgIGlmIGhhc2F0dHIoc2VsZWN0X2Zyb21fbW9kZWwuZXN0aW1hdG9yXywgImNvZWZfIik6CiAgICAgICAgICAgIHN0YXRfZGYgPSBzZWxlY3RfZnJvbV9tb2RlbC5lc3RpbWF0b3JfLmNvZWZfCiAgICAgICAgZWxpZiBoYXNhdHRyKHNlbGVjdF9mcm9tX21vZGVsLmVzdGltYXRvcl8sICJmZWF0dXJlX2ltcG9ydGFuY2VzXyIpOgogICAgICAgICAgICBzdGF0X2RmID0gc2VsZWN0X2Zyb21fbW9kZWwuZXN0aW1hdG9yXy5mZWF0dXJlX2ltcG9ydGFuY2VzXwoKICAgICAgICBzdGF0X2RmID0gcGQuRGF0YUZyYW1lKGluZGV4PVguY29sdW1ucywgY29sdW1ucz1bbW9kZWxfbmFtZV0sIGRhdGE9c3RhdF9kZlswXSkKICAgICAgICBtb2RlbHNfZGYgPSBtb2RlbHNfZGYuam9pbihzdGF0X2RmKQoKICAgICAgICBwbG90X3N0YXQoY29udGV4dCwgbW9kZWxfbmFtZSwgc3RhdF9kZikKCiAgICAjIENyZWF0ZSBmZWF0dXJlX3Njb3JlcyBERiB3aXRoIHN0YXQgJiBtb2RlbCBmaWx0ZXJzIHNjb3JlcwogICAgcmVzdWx0X21hdHJpeF9kZiA9IHBkLmNvbmNhdChbc3RhdHNfZGYsIG1vZGVsc19kZl0sIGF4aXM9MSwgc29ydD1GYWxzZSkKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoCiAgICAgICAga2V5PSJmZWF0dXJlX3Njb3JlcyIsCiAgICAgICAgZGY9cmVzdWx0X21hdHJpeF9kZiwKICAgICAgICBsb2NhbF9wYXRoPSJmZWF0dXJlX3Njb3Jlcy5wYXJxdWV0IiwKICAgICAgICBmb3JtYXQ9InBhcnF1ZXQiLAogICAgKQogICAgaWYgbWF4X3NjYWxlZF9zY29yZXM6CiAgICAgICAgbm9ybWFsaXplZF9kZiA9IHJlc3VsdF9tYXRyaXhfZGYucmVwbGFjZShbbnAuaW5mLCAtbnAuaW5mXSwgbnAubmFuKS52YWx1ZXMKICAgICAgICBtaW5fbWF4X3NjYWxlciA9IE1pbk1heFNjYWxlcigpCiAgICAgICAgbm9ybWFsaXplZF9kZiA9IG1pbl9tYXhfc2NhbGVyLmZpdF90cmFuc2Zvcm0obm9ybWFsaXplZF9kZikKICAgICAgICBub3JtYWxpemVkX2RmID0gcGQuRGF0YUZyYW1lKAogICAgICAgICAgICBkYXRhPW5vcm1hbGl6ZWRfZGYsCiAgICAgICAgICAgIGNvbHVtbnM9cmVzdWx0X21hdHJpeF9kZi5jb2x1bW5zLAogICAgICAgICAgICBpbmRleD1yZXN1bHRfbWF0cml4X2RmLmluZGV4LAogICAgICAgICkKICAgICAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgICAgICBrZXk9Im1heF9zY2FsZWRfc2NvcmVzX2ZlYXR1cmVfc2NvcmVzIiwKICAgICAgICAgICAgZGY9bm9ybWFsaXplZF9kZiwKICAgICAgICAgICAgbG9jYWxfcGF0aD0ibWF4X3NjYWxlZF9zY29yZXNfZmVhdHVyZV9zY29yZXMucGFycXVldCIsCiAgICAgICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICAgICAgKQoKICAgICMgQ3JlYXRlIGZlYXR1cmUgY291bnQgRGF0YUZyYW1lCiAgICBmb3IgdGVzdF9uYW1lIGluIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZzoKICAgICAgICByZXN1bHRfbWF0cml4X2RmW3Rlc3RfbmFtZV0gPSBbCiAgICAgICAgICAgIDEgaWYgeCBpbiBzZWxlY3RlZF9mZWF0dXJlc19hZ2dbdGVzdF9uYW1lXSBlbHNlIDAgZm9yIHggaW4gWC5jb2x1bW5zCiAgICAgICAgXQogICAgcmVzdWx0X21hdHJpeF9kZi5sb2NbOiwgIm51bV92b3RlcyJdID0gcmVzdWx0X21hdHJpeF9kZi5zdW0oYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICBrZXk9InNlbGVjdGVkX2ZlYXR1cmVzX2NvdW50IiwKICAgICAgICBkZj1yZXN1bHRfbWF0cml4X2RmLAogICAgICAgIGxvY2FsX3BhdGg9InNlbGVjdGVkX2ZlYXR1cmVzX2NvdW50LnBhcnF1ZXQiLAogICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICApCgogICAgIyBIb3cgbWFueSB2b3RlcyBhcmUgbmVlZGVkIGZvciBhIGZlYXR1cmUgdG8gYmUgc2VsZWN0ZWQ/CiAgICBpZiBpc2luc3RhbmNlKG1pbl92b3RlcywgaW50KToKICAgICAgICB2b3Rlc19uZWVkZWQgPSBtaW5fdm90ZXMKICAgIGVsc2U6CiAgICAgICAgbnVtX2ZpbHRlcnMgPSBsZW4oc3RhdF9maWx0ZXJzKSArIGxlbihtb2RlbF9maWx0ZXJzKQogICAgICAgIHZvdGVzX25lZWRlZCA9IGludChucC5mbG9vcihudW1fZmlsdGVycyAqIG1heChtaW4obWluX3ZvdGVzLCAxKSwgMCkpKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInZvdGVzIG5lZWRlZCB0byBiZSBzZWxlY3RlZDoge3ZvdGVzX25lZWRlZH0iKQoKICAgICMgQ3JlYXRlIGZpbmFsIGZlYXR1cmUgZGF0YWZyYW1lCiAgICBzZWxlY3RlZF9mZWF0dXJlcyA9IHJlc3VsdF9tYXRyaXhfZGZbCiAgICAgICAgcmVzdWx0X21hdHJpeF9kZi5udW1fdm90ZXMgPj0gdm90ZXNfbmVlZGVkCiAgICBdLmluZGV4LnRvbGlzdCgpCiAgICBnb29kX2ZlYXR1cmVfZGYgPSBkZi5sb2NbOiwgc2VsZWN0ZWRfZmVhdHVyZXNdCiAgICBmaW5hbF9kZiA9IHBkLmNvbmNhdChbZ29vZF9mZWF0dXJlX2RmLCB5XSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICBrZXk9InNlbGVjdGVkX2ZlYXR1cmVzIiwKICAgICAgICBkZj1maW5hbF9kZiwKICAgICAgICBsb2NhbF9wYXRoPSJzZWxlY3RlZF9mZWF0dXJlcy5wYXJxdWV0IiwKICAgICAgICBmb3JtYXQ9InBhcnF1ZXQiLAogICAgKQoKICAgICMgQ3JlYXRpbmcgYSBuZXcgZmVhdHVyZSB2ZWN0b3IgY29udGFpbmluZyBvbmx5IHRoZSBpZGVudGlmaWVkIHRvcCBmZWF0dXJlcwogICAgaWYgaXNfZmVhdHVyZV92ZWN0b3IgYW5kIGRmX2FydGlmYWN0Lm1ldGEuc3BlYy5mZWF0dXJlcyBhbmQgb3V0cHV0X3ZlY3Rvcl9uYW1lOgogICAgICAgICMgU2VsZWN0aW5nIHRoZSB0b3AgSyBmZWF0dXJlcyBmcm9tIG91ciB0b3AgZmVhdHVyZSBkYXRhZnJhbWUKICAgICAgICBzZWxlY3RlZF9mZWF0dXJlcyA9IHJlc3VsdF9tYXRyaXhfZGYuaGVhZChrKS5pbmRleAoKICAgICAgICAjIE1hdGNoIHRoZSBzZWxlY3RlZCBmZWF0dXJlIG5hbWVzIHRvIHRoZSBGUyBGZWF0dXJlIGFubm90YXRpb25zCiAgICAgICAgbWF0Y2hlZF9zZWxlY3Rpb25zID0gWwogICAgICAgICAgICBmZWF0dXJlCiAgICAgICAgICAgIGZvciBmZWF0dXJlIGluIGxpc3QoZGZfYXJ0aWZhY3QubWV0YS5zcGVjLmZlYXR1cmVzKQogICAgICAgICAgICBmb3Igc2VsZWN0ZWQgaW4gbGlzdChzZWxlY3RlZF9mZWF0dXJlcykKICAgICAgICAgICAgaWYgZmVhdHVyZS5lbmRzd2l0aChzZWxlY3RlZCkKICAgICAgICBdCgogICAgICAgICMgRGVmaW5pbmcgb3VyIG5ldyBmZWF0dXJlIHZlY3RvcgogICAgICAgIHRvcF9mZWF0dXJlc19mdiA9IGZzLkZlYXR1cmVWZWN0b3IoCiAgICAgICAgICAgIG91dHB1dF92ZWN0b3JfbmFtZSwKICAgICAgICAgICAgbWF0Y2hlZF9zZWxlY3Rpb25zLAogICAgICAgICAgICBsYWJlbF9mZWF0dXJlPSJsYWJlbHMubGFiZWwiLAogICAgICAgICAgICBkZXNjcmlwdGlvbj0iZmVhdHVyZSB2ZWN0b3IgY29tcG9zZWQgc3RyaWN0bHkgb2Ygb3VyIHRvcCBmZWF0dXJlcyIsCiAgICAgICAgKQoKICAgICAgICAjIFNhdmluZwogICAgICAgIHRvcF9mZWF0dXJlc19mdi5zYXZlKCkKICAgICAgICBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcyh0b3BfZmVhdHVyZXNfZnYsIHRhcmdldD1QYXJxdWV0VGFyZ2V0KCkpCgogICAgICAgICMgTG9nZ2luZyBvdXIgbmV3IGZlYXR1cmUgdmVjdG9yIFVSSQogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdCgidG9wX2ZlYXR1cmVzX3ZlY3RvciIsIHRvcF9mZWF0dXJlc19mdi51cmkpCg== code_origin: '' default_handler: feature_selection - description: Select features through multiple Statistical and Model filters image: mlrun/mlrun -metadata: - categories: - - data-preparation - - machine-learning - tag: '' - name: feature-selection + description: Select features through multiple Statistical and Model filters +verbose: false diff --git a/feature_selection/item.yaml b/feature_selection/item.yaml index 1c79e6f12..c7400a7f0 100644 --- a/feature_selection/item.yaml +++ b/feature_selection/item.yaml @@ -22,4 +22,4 @@ spec: kind: job requirements: [] url: '' -version: 1.5.0 +version: 1.7.0 From d1d61a0f1765749cdd5ee0d4225aea1c60093f88 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Mon, 2 Sep 2024 16:09:01 +0300 Subject: [PATCH 4/8] Revert "[onnx utils] update onnx utils packages" This reverts commit 88727986ffa91662593958023be8ac3ccef2cab0. --- onnx_utils/function.yaml | 42 ++++++++++++++++++----------------- onnx_utils/item.yaml | 16 ++++++------- onnx_utils/requirements.txt | 12 +++++----- onnx_utils/test_onnx_utils.py | 4 ++-- 4 files changed, 38 insertions(+), 36 deletions(-) diff --git a/onnx_utils/function.yaml b/onnx_utils/function.yaml index 88f810fb4..7a0054c4d 100644 --- a/onnx_utils/function.yaml +++ b/onnx_utils/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: onnx-utils tag: '' - hash: fd6cd909ef6e055c348b44a0313e190513cd755b + hash: 0c4a6491b976d5220d3ebfb83a3905dd47e86be2 project: '' labels: author: guyl @@ -16,16 +16,16 @@ spec: functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgQ2FsbGFibGUsIERpY3QsIExpc3QsIFR1cGxlCgppbXBvcnQgbWxydW4KCgpjbGFzcyBfVG9PTk5YQ29udmVyc2lvbnM6CiAgICAiIiIKICAgIEFuIE9OTlggY29udmVyc2lvbiBmdW5jdGlvbnMgbGlicmFyeSBjbGFzcy4KICAgICIiIgoKICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiB0Zl9rZXJhc190b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50XSwgc3RyXV0gPSBOb25lLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBDb252ZXJ0IGEgVEYuS2VyYXMgbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICBBbiBpbml0aWFsaXplZCBURktlcmFzTW9kZWxIYW5kbGVyIHdpdGggYSBsb2FkZWQgbW9kZWwgdG8gY29udmVydCB0byBPTk5YLgogICAgICAgIDpwYXJhbSBvbm54X21vZGVsX25hbWU6IFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICBXaGV0aGVyIG9yIG5vdCB0byBvcHRpbWl6ZSB0aGUgT05OWCBtb2RlbCB1c2luZyAnb25ueG9wdGltaXplcicgYmVmb3JlIHNhdmluZyB0aGUgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgRGVmYXVsdGVkIHRvIFRydWUuCiAgICAgICAgOnBhcmFtIGlucHV0X3NpZ25hdHVyZTogQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEgbGlzdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdoZXJlIGVhY2ggZWxlbWVudCBpcyBhbiBpbnB1dCBsYXllciB0dXBsZS4gQW4gaW5wdXQgbGF5ZXIgdHVwbGUgaXMgYSB0dXBsZSBvZjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMF0gPSBMYXllcidzIHNoYXBlLCBhIHR1cGxlIG9mIGludGVnZXJzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBOb25lLCB0aGUgaW5wdXQgc2lnbmF0dXJlIHdpbGwgYmUgdHJpZWQgdG8gYmUgcmVhZCBmcm9tIHRoZSBtb2RlbCBhcnRpZmFjdC4gRGVmYXVsdGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG8gTm9uZS4KICAgICAgICAiIiIKICAgICAgICAjIEltcG9ydCB0aGUgZnJhbWV3b3JrIGFuZCBoYW5kbGVyOgogICAgICAgIGltcG9ydCB0ZW5zb3JmbG93IGFzIHRmCiAgICAgICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLnRmX2tlcmFzIGltcG9ydCBURktlcmFzVXRpbHMKCiAgICAgICAgIyBDaGVjayB0aGUgZ2l2ZW4gJ2lucHV0X3NpZ25hdHVyZScgcGFyYW1ldGVyOgogICAgICAgIGlmIGlucHV0X3NpZ25hdHVyZSBpcyBOb25lOgogICAgICAgICAgICAjIFJlYWQgdGhlIGlucHV0cyBmcm9tIHRoZSBtb2RlbDoKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgbW9kZWxfaGFuZGxlci5yZWFkX2lucHV0c19mcm9tX21vZGVsKCkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlcnJvcjoKICAgICAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1blJ1bnRpbWVFcnJvcigKICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSBwcm92aWRlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXIuIFRoZSBmdW5jdGlvbiB0cmllZCByZWFkaW5nIHRoZSBpbnB1dCBsYXllcnMgIgogICAgICAgICAgICAgICAgICAgIGYiaW5mb3JtYXRpb24gYXV0b21hdGljYWxseSBidXQgZmFpbGVkIHdpdGggdGhlIGZvbGxvd2luZyBlcnJvcjoge2Vycm9yfSIKICAgICAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICAjIFBhcnNlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXI6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IFsKICAgICAgICAgICAgICAgIHRmLlRlbnNvclNwZWMoCiAgICAgICAgICAgICAgICAgICAgc2hhcGU9c2hhcGUsCiAgICAgICAgICAgICAgICAgICAgZHR5cGU9VEZLZXJhc1V0aWxzLmNvbnZlcnRfdmFsdWVfdHlwZV90b190Zl9kdHlwZSgKICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICBdCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICkKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgcHl0b3JjaF90b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50LCAuLi5dLCBzdHJdXSA9IE5vbmUsCiAgICAgICAgaW5wdXRfbGF5ZXJzX25hbWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIG91dHB1dF9sYXllcnNfbmFtZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgZHluYW1pY19heGVzOiBEaWN0W3N0ciwgRGljdFtpbnQsIHN0cl1dID0gTm9uZSwKICAgICAgICBpc19iYXRjaGVkOiBib29sID0gVHJ1ZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBhIFB5VG9yY2ggbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICAgICAgQW4gaW5pdGlhbGl6ZWQgUHlUb3JjaE1vZGVsSGFuZGxlciB3aXRoIGEgbG9hZGVkIG1vZGVsIHRvIGNvbnZlcnQgdG8gT05OWC4KICAgICAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgICAgVGhlIG5hbWUgdG8gdXNlIHRvIGxvZyB0aGUgY29udmVydGVkIE9OTlggbW9kZWwuIElmIG5vdCBnaXZlbiwgdGhlIGdpdmVuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtb2RlbF9uYW1lYCB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgICAgV2hldGhlciBvciBub3QgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgICAgICA6cGFyYW0gaW5wdXRfc2lnbmF0dXJlOiAgICAgQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdCB3aGVyZSBlYWNoIGVsZW1lbnQgaXMgYW4gaW5wdXQgbGF5ZXIgdHVwbGUuIEFuIGlucHV0IGxheWVyIHR1cGxlIGlzIGEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFswXSA9IExheWVyJ3Mgc2hhcGUsIGEgdHVwbGUgb2YgaW50ZWdlcnMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgdGhlIGlucHV0IHNpZ25hdHVyZSB3aWxsIGJlIHRyaWVkIHRvIGJlIHJlYWQgZnJvbSB0aGUgbW9kZWwgYXJ0aWZhY3QuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpbnB1dF9sYXllcnNfbmFtZXM6ICBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgaW5wdXQgbm9kZXMgb2YgdGhlIGdyYXBoIGluIG9yZGVyLiBBbGwgb2YgdGhlIG90aGVyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlcnMgKGlubmVyIGxheWVycykgY2FuIGJlIHNldCBhcyB3ZWxsIGJ5IHBhc3NpbmcgYWRkaXRpb25hbCBuYW1lcyBpbiB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdC4gVGhlIG9yZGVyIGlzIGJ5IHRoZSBvcmRlciBvZiB0aGUgcGFyYW1ldGVycyBpbiB0aGUgbW9kZWwuIElmIE5vbmUsIHRoZSBpbnB1dHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBpbnB1dHMuIElmIGl0cyBhbHNvIE5vbmUsIGl0IGlzIGRlZmF1bHRlZCB0bzoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlucHV0XzAiLCAiaW5wdXRfMSIsIC4uLgogICAgICAgIDpwYXJhbSBvdXRwdXRfbGF5ZXJzX25hbWVzOiBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgb3V0cHV0IG5vZGVzIG9mIHRoZSBncmFwaCBpbiBvcmRlci4gSWYgTm9uZSwgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG91dHB1dHMgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBvdXRwdXRzLiBJZiBpdHMgYWxzbyBOb25lLCBpdCBpcyBkZWZhdWx0ZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG86ICJvdXRwdXRfMCIgKGZvciBtdWx0aXBsZSBvdXRwdXRzLCB0aGlzIHBhcmFtZXRlciBtdXN0IGJlIHByb3ZpZGVkKS4KICAgICAgICA6cGFyYW0gZHluYW1pY19heGVzOiAgICAgICAgSWYgcGFydCBvZiB0aGUgaW5wdXQgLyBvdXRwdXQgc2hhcGUgaXMgZHluYW1pYywgbGlrZSAoYmF0Y2hfc2l6ZSwgMywgMzIsIDMyKSB5b3UgY2FuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNwZWNpZnkgaXQgYnkgZ2l2aW5nIGEgZHluYW1pYyBheGlzIHRvIHRoZSBpbnB1dCAvIG91dHB1dCBsYXllciBieSBpdHMgbmFtZSBhcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb2xsb3dzOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiaW5wdXQgbGF5ZXIgbmFtZSI6IHswOiAiYmF0Y2hfc2l6ZSJ9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm91dHB1dCBsYXllciBuYW1lIjogezA6ICJiYXRjaF9zaXplIn0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgcHJvdmlkZWQsIHRoZSAnaXNfYmF0Y2hlZCcgZmxhZyB3aWxsIGJlIGlnbm9yZWQuIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpc19iYXRjaGVkOiAgICAgICAgICBXaGV0aGVyIHRvIGluY2x1ZGUgYSBiYXRjaCBzaXplIGFzIHRoZSBmaXJzdCBheGlzIGluIGV2ZXJ5IGlucHV0IGFuZCBvdXRwdXQgbGF5ZXIuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBUcnVlLiBXaWxsIGJlIGlnbm9yZWQgaWYgJ2R5bmFtaWNfYXhlcycgaXMgcHJvdmlkZWQuCiAgICAgICAgIiIiCiAgICAgICAgIyBJbXBvcnQgdGhlIGZyYW1ld29yayBhbmQgaGFuZGxlcjoKICAgICAgICBpbXBvcnQgdG9yY2gKICAgICAgICBmcm9tIG1scnVuLmZyYW1ld29ya3MucHl0b3JjaCBpbXBvcnQgUHlUb3JjaFV0aWxzCgogICAgICAgICMgUGFyc2UgdGhlICdpbnB1dF9zaWduYXR1cmUnIHBhcmFtZXRlcjoKICAgICAgICBpZiBpbnB1dF9zaWduYXR1cmUgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IHR1cGxlKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRvcmNoLnplcm9zKAogICAgICAgICAgICAgICAgICAgICAgICBzaXplPXNoYXBlLAogICAgICAgICAgICAgICAgICAgICAgICBkdHlwZT1QeVRvcmNoVXRpbHMuY29udmVydF92YWx1ZV90eXBlX3RvX3RvcmNoX2R0eXBlKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICApCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NhbXBsZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICAgICBpbnB1dF9sYXllcnNfbmFtZXM9aW5wdXRfbGF5ZXJzX25hbWVzLAogICAgICAgICAgICBvdXRwdXRfbGF5ZXJzX25hbWVzPW91dHB1dF9sYXllcnNfbmFtZXMsCiAgICAgICAgICAgIGR5bmFtaWNfYXhlcz1keW5hbWljX2F4ZXMsCiAgICAgICAgICAgIGlzX2JhdGNoZWQ9aXNfYmF0Y2hlZAogICAgICAgICkKCgojIE1hcCBmb3IgZ2V0dGluZyB0aGUgY29udmVyc2lvbiBmdW5jdGlvbiBhY2NvcmRpbmcgdG8gdGhlIHByb3ZpZGVkIGZyYW1ld29yazoKX0NPTlZFUlNJT05fTUFQID0gewogICAgInRlbnNvcmZsb3cua2VyYXMiOiBfVG9PTk5YQ29udmVyc2lvbnMudGZfa2VyYXNfdG9fb25ueCwKICAgICJ0b3JjaCI6IF9Ub09OTlhDb252ZXJzaW9ucy5weXRvcmNoX3RvX29ubngsCn0gICMgdHlwZTogRGljdFtzdHIsIENhbGxhYmxlXQoKCmRlZiB0b19vbm54KAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBvbm54X21vZGVsX25hbWU6IHN0ciA9IE5vbmUsCiAgICBvcHRpbWl6ZV9tb2RlbDogYm9vbCA9IFRydWUsCiAgICBmcmFtZXdvcmtfa3dhcmdzOiBEaWN0W3N0ciwgQW55XSA9IE5vbmUsCik6CiAgICAiIiIKICAgIENvbnZlcnQgdGhlIGdpdmVuIG1vZGVsIHRvIGFuIE9OTlggbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgIFRoZSBNTFJ1biBmdW5jdGlvbiBleGVjdXRpb24gY29udGV4dAogICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgIFRoZSBtb2RlbCBwYXRoIHN0b3JlIG9iamVjdC4KICAgIDpwYXJhbSBvbm54X21vZGVsX25hbWU6ICBUaGUgbmFtZSB0byB1c2UgdG8gbG9nIHRoZSBjb252ZXJ0ZWQgT05OWCBtb2RlbC4gSWYgbm90IGdpdmVuLCB0aGUgZ2l2ZW4gYG1vZGVsX25hbWVgIHdpbGwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkIHdpdGggYW4gYWRkaXRpb25hbCBzdWZmaXggYF9vbm54YC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgV2hldGhlciB0byBvcHRpbWl6ZSB0aGUgT05OWCBtb2RlbCB1c2luZyAnb25ueG9wdGltaXplcicgYmVmb3JlIHNhdmluZyB0aGUgbW9kZWwuIERlZmF1bHRlZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRvIFRydWUuCiAgICA6cGFyYW0gZnJhbWV3b3JrX2t3YXJnczogQWRkaXRpb25hbCBhcmd1bWVudHMgZWFjaCBmcmFtZXdvcmsgbWF5IHJlcXVpcmUgaW4gb3JkZXIgdG8gY29udmVydCB0byBPTk5YLiBUbyBnZXQgdGhlIGRvYwogICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0cmluZyBvZiB0aGUgZGVzaXJlZCBmcmFtZXdvcmsgb25ueCBjb252ZXJzaW9uIGZ1bmN0aW9uLCBwYXNzICJoZWxwIi4KICAgICIiIgogICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLmF1dG9fbWxydW4uYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCgogICAgIyBHZXQgYSBtb2RlbCBoYW5kbGVyIG9mIHRoZSByZXF1aXJlZCBmcmFtZXdvcms6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbF9wYXRoLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBHZXQgdGhlIG1vZGVsJ3MgZnJhbWV3b3JrOgogICAgZnJhbWV3b3JrID0gbW9kZWxfaGFuZGxlci5GUkFNRVdPUktfTkFNRQoKICAgICMgVXNlIHRoZSBjb252ZXJzaW9uIG1hcCB0byBnZXQgdGhlIHNwZWNpZmljIGZyYW1ld29yayB0byBvbm54IGNvbnZlcnNpb246CiAgICBpZiBmcmFtZXdvcmsgbm90IGluIF9DT05WRVJTSU9OX01BUDoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJUaGUgZm9sbG93aW5nIGZyYW1ld29yazogJ3tmcmFtZXdvcmt9JywgaGFzIG5vIE9OTlggY29udmVyc2lvbi4iCiAgICAgICAgKQogICAgY29udmVyc2lvbl9mdW5jdGlvbiA9IF9DT05WRVJTSU9OX01BUFtmcmFtZXdvcmtdCgogICAgIyBDaGVjayBpZiBuZWVkZWQgdG8gcHJpbnQgdGhlIGZ1bmN0aW9uJ3MgZG9jIHN0cmluZyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzID09ICJoZWxwIjoKICAgICAgICBwcmludChjb252ZXJzaW9uX2Z1bmN0aW9uLl9fZG9jX18pCiAgICAgICAgcmV0dXJuCgogICAgIyBTZXQgdGhlIGRlZmF1bHQgZW1wdHkgZnJhbWV3b3JrIGt3YXJncyBpZiBuZWVkZWQ6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzIGlzIE5vbmU6CiAgICAgICAgZnJhbWV3b3JrX2t3YXJncyA9IHt9CgogICAgIyBSdW4gdGhlIGNvbnZlcnNpb246CiAgICB0cnk6CiAgICAgICAgY29udmVyc2lvbl9mdW5jdGlvbigKICAgICAgICAgICAgbW9kZWxfaGFuZGxlcj1tb2RlbF9oYW5kbGVyLAogICAgICAgICAgICBvbm54X21vZGVsX25hbWU9b25ueF9tb2RlbF9uYW1lLAogICAgICAgICAgICBvcHRpbWl6ZV9tb2RlbD1vcHRpbWl6ZV9tb2RlbCwKICAgICAgICAgICAgKipmcmFtZXdvcmtfa3dhcmdzLAogICAgICAgICkKICAgIGV4Y2VwdCBUeXBlRXJyb3IgYXMgZXhjZXB0aW9uOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIkVSUk9SOiBBIFR5cGVFcnJvciBleGNlcHRpb24gd2FzIHJhaXNlZCBkdXJpbmcgdGhlIGNvbnZlcnNpb246XG57ZXhjZXB0aW9ufS4gIgogICAgICAgICAgICBmIlBsZWFzZSByZWFkIHRoZSB7ZnJhbWV3b3JrfSBmcmFtZXdvcmsgY29udmVyc2lvbiBmdW5jdGlvbiBkb2Mgc3RyaW5nIGJ5IHBhc3NpbmcgJ2hlbHAnIGluIHRoZSAiCiAgICAgICAgICAgIGYiJ2ZyYW1ld29ya19rd2FyZ3MnIGRpY3Rpb25hcnkgcGFyYW1ldGVyLiIKICAgICAgICApCgoKZGVmIG9wdGltaXplKAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBvcHRpbWl6YXRpb25zOiBMaXN0W3N0cl0gPSBOb25lLAogICAgZml4ZWRfcG9pbnQ6IGJvb2wgPSBGYWxzZSwKICAgIG9wdGltaXplZF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAopOgogICAgIiIiCiAgICBPcHRpbWl6ZSB0aGUgZ2l2ZW4gT05OWCBtb2RlbC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgIFRoZSBNTFJ1biBmdW5jdGlvbiBleGVjdXRpb24gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbF9wYXRoOiAgICAgICAgICAgUGF0aCB0byB0aGUgT05OWCBtb2RlbCBvYmplY3QuCiAgICA6cGFyYW0gb3B0aW1pemF0aW9uczogICAgICAgIExpc3Qgb2YgcG9zc2libGUgb3B0aW1pemF0aW9ucy4gVG8gc2VlIHdoYXQgb3B0aW1pemF0aW9ucyBhcmUgYXZhaWxhYmxlLCBwYXNzICJoZWxwIi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgYWxsIG9mIHRoZSBvcHRpbWl6YXRpb25zIHdpbGwgYmUgdXNlZC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gZml4ZWRfcG9pbnQ6ICAgICAgICAgIE9wdGltaXplIHRoZSB3ZWlnaHRzIHVzaW5nIGZpeGVkIHBvaW50LiBEZWZhdWx0ZWQgdG8gRmFsc2UuCiAgICA6cGFyYW0gb3B0aW1pemVkX21vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBvcHRpbWl6ZWQgbW9kZWwuIElmIE5vbmUsIHRoZSBvcmlnaW5hbCBtb2RlbCB3aWxsIGJlIG92ZXJyaWRkZW4uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgIiIiCiAgICAjIEltcG9ydCB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGltcG9ydCBvbm54b3B0aW1pemVyCiAgICBmcm9tIG1scnVuLmZyYW1ld29ya3Mub25ueCBpbXBvcnQgT05OWE1vZGVsSGFuZGxlcgoKICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIHByaW50IHRoZSBhdmFpbGFibGUgb3B0aW1pemF0aW9ucyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBvcHRpbWl6YXRpb25zID09ICJoZWxwIjoKICAgICAgICBhdmFpbGFibGVfcGFzc2VzID0gIlxuKiAiLmpvaW4ob25ueG9wdGltaXplci5nZXRfYXZhaWxhYmxlX3Bhc3NlcygpKQogICAgICAgIHByaW50KGYiVGhlIGF2YWlsYWJsZSBvcHRpbWl6YXRpb25zIGFyZTpcbioge2F2YWlsYWJsZV9wYXNzZXN9IikKICAgICAgICByZXR1cm4KCiAgICAjIENyZWF0ZSB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBPTk5YTW9kZWxIYW5kbGVyKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWxfcGF0aCwgY29udGV4dD1jb250ZXh0CiAgICApCgogICAgIyBMb2FkIHRoZSBPTk5YIG1vZGVsOgogICAgbW9kZWxfaGFuZGxlci5sb2FkKCkKCiAgICAjIE9wdGltaXplIHRoZSBtb2RlbCB1c2luZyB0aGUgZ2l2ZW4gY29uZmlndXJhdGlvbnM6CiAgICBtb2RlbF9oYW5kbGVyLm9wdGltaXplKG9wdGltaXphdGlvbnM9b3B0aW1pemF0aW9ucywgZml4ZWRfcG9pbnQ9Zml4ZWRfcG9pbnQpCgogICAgIyBSZW5hbWUgaWYgbmVlZGVkOgogICAgaWYgb3B0aW1pemVkX21vZGVsX25hbWUgaXMgbm90IE5vbmU6CiAgICAgICAgbW9kZWxfaGFuZGxlci5zZXRfbW9kZWxfbmFtZShtb2RlbF9uYW1lPW9wdGltaXplZF9tb2RlbF9uYW1lKQoKICAgICMgTG9nIHRoZSBvcHRpbWl6ZWQgbW9kZWw6CiAgICBtb2RlbF9oYW5kbGVyLmxvZygpCg== base_image: mlrun/mlrun commands: [] - code_origin: '' - origin_filename: '' + code_origin: https://github.com/yonishelach/functions.git#f84b9565a33d8159315992ebba5838d41f6cc112:/Users/Yonatan_Shelach/projects/functions/onnx_utils/onnx_utils.py + origin_filename: /Users/Yonatan_Shelach/projects/functions/onnx_utils/onnx_utils.py with_mlrun: false auto_build: true requirements: - - onnx~=1.15.0 - - onnxruntime~=1.8.1 - - onnxoptimizer~=0.2.0 - - onnxmltools~=1.9.0 - - tf2onnx~=1.16.0 + - onnx~=1.13.0 + - onnxruntime~=1.14.0 + - onnxoptimizer~=0.3.0 + - onnxmltools~=1.11.0 + - tf2onnx~=1.13.0 entry_points: tf_keras_to_onnx: name: tf_keras_to_onnx @@ -35,6 +35,7 @@ spec: - name: model_handler doc: An initialized TFKerasModelHandler with a loaded model to convert to ONNX. + default: '' - name: onnx_model_name type: str doc: The name to use to log the converted ONNX model. If not given, the given @@ -54,10 +55,9 @@ spec: data type, a mlrun.data_types.ValueType string. If None, the input signature will be tried to be read from the model artifact. Defaulted to None.' default: null - outputs: [] + outputs: + - default: '' lineno: 26 - has_varargs: false - has_kwargs: false pytorch_to_onnx: name: pytorch_to_onnx doc: Convert a PyTorch model to an ONNX model and log it back to MLRun as a @@ -66,6 +66,7 @@ spec: - name: model_handler doc: An initialized PyTorchModelHandler with a loaded model to convert to ONNX. + default: '' - name: onnx_model_name type: str doc: The name to use to log the converted ONNX model. If not given, the given @@ -113,10 +114,9 @@ spec: doc: Whether to include a batch size as the first axis in every input and output layer. Defaulted to True. Will be ignored if 'dynamic_axes' is provided. default: true - outputs: [] + outputs: + - default: '' lineno: 81 - has_varargs: false - has_kwargs: false to_onnx: name: to_onnx doc: Convert the given model to an ONNX model. @@ -124,9 +124,11 @@ spec: - name: context type: MLClientCtx doc: The MLRun function execution context + default: '' - name: model_path type: str doc: The model path store object. + default: '' - name: onnx_model_name type: str doc: The name to use to log the converted ONNX model. If not given, the given @@ -144,10 +146,9 @@ spec: ONNX. To get the doc string of the desired framework onnx conversion function, pass "help". default: null - outputs: [] + outputs: + - default: '' lineno: 160 - has_varargs: false - has_kwargs: false optimize: name: optimize doc: Optimize the given ONNX model. @@ -155,9 +156,11 @@ spec: - name: context type: MLClientCtx doc: The MLRun function execution context. + default: '' - name: model_path type: str doc: Path to the ONNX model object. + default: '' - name: optimizations type: List[str] doc: List of possible optimizations. To see what optimizations are available, @@ -173,10 +176,9 @@ spec: doc: The name of the optimized model. If None, the original model will be overridden. Defaulted to None. default: null - outputs: [] + outputs: + - default: '' lineno: 219 - has_varargs: false - has_kwargs: false description: ONNX intigration in MLRun, some utils functions for the ONNX framework, optimizing and converting models from different framework to ONNX using MLRun. default_handler: to_onnx diff --git a/onnx_utils/item.yaml b/onnx_utils/item.yaml index 84486d9f8..36335837a 100644 --- a/onnx_utils/item.yaml +++ b/onnx_utils/item.yaml @@ -12,9 +12,9 @@ labels: author: guyl maintainers: [] marketplaceType: '' -mlrunVersion: 1.6.3 +mlrunVersion: 1.1.0 name: onnx_utils -platformVersion: 3.5.4 +platformVersion: 3.5.0 spec: extra_spec: allow_empty_resources: true @@ -26,10 +26,10 @@ spec: image: mlrun/mlrun kind: job requirements: - - onnx~=1.15.0 - - onnxruntime~=1.8.1 - - onnxoptimizer~=0.2.0 - - onnxmltools~=1.9.0 - - tf2onnx~=1.16.0 + - onnx~=1.13.0 + - onnxruntime~=1.14.0 + - onnxoptimizer~=0.3.0 + - onnxmltools~=1.11.0 + - tf2onnx~=1.13.0 url: '' -version: 1.3.0 +version: 1.2.0 diff --git a/onnx_utils/requirements.txt b/onnx_utils/requirements.txt index a9acd7371..dc7ff1e7b 100644 --- a/onnx_utils/requirements.txt +++ b/onnx_utils/requirements.txt @@ -1,11 +1,11 @@ tqdm~=4.62.3 -tensorflow~=2.13.0 +tensorflow~=2.7.0 torch~=1.10.0 torchvision~=0.11.1 -onnx~=1.15.0 -onnxruntime~=1.12.1 -onnxoptimizer~=0.3.0 +onnx~=1.10.1 +onnxruntime~=1.8.1 +onnxoptimizer~=0.2.0 onnxmltools~=1.9.0 -tf2onnx~=1.16.0 +tf2onnx~=1.9.0 plotly~=5.4.0 -#wrapt<1.15.0 # wrapt==1.15.0 fails tensorflow 2.7 Todo: please remove when updating tensorflow \ No newline at end of file +wrapt<1.15.0 # wrapt==1.15.0 fails tensorflow 2.7 Todo: please remove when updating tensorflow \ No newline at end of file diff --git a/onnx_utils/test_onnx_utils.py b/onnx_utils/test_onnx_utils.py index aaae96372..35b224c4a 100644 --- a/onnx_utils/test_onnx_utils.py +++ b/onnx_utils/test_onnx_utils.py @@ -257,7 +257,7 @@ def test_pytorch_to_onnx(): filename="test_onnx_utils.py", name="log_model", kind="job", - image="mlrun/mlrun", + image="mlrun/ml-models", ) # Run the function to log the model: @@ -341,7 +341,7 @@ def test_optimize(): filename="test_onnx_utils.py", name="log_model", kind="job", - image="mlrun/mlrun", + image="mlrun/ml-models", ) # Run the function to log the model: From 98d09427db1cc9bdd08e3c498289e0bb35664df6 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Mon, 2 Sep 2024 16:12:12 +0300 Subject: [PATCH 5/8] [feature selection] update function yaml --- feature_selection/item.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_selection/item.yaml b/feature_selection/item.yaml index c7400a7f0..1b25ec410 100644 --- a/feature_selection/item.yaml +++ b/feature_selection/item.yaml @@ -12,7 +12,7 @@ labels: author: orz maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.6.4 name: feature-selection platformVersion: 3.6.0 spec: From 2fb20802033039178fcf94ad33eff63155e9ff37 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Mon, 2 Sep 2024 16:34:04 +0300 Subject: [PATCH 6/8] [feature selection] update function yaml --- feature_selection/requirements.txt | 2 +- feature_selection/test_feature_selection.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/feature_selection/requirements.txt b/feature_selection/requirements.txt index 961f64ea4..70a079c7d 100644 --- a/feature_selection/requirements.txt +++ b/feature_selection/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn~=1.0.2 +scikit-learn matplotlib seaborn scikit-plot diff --git a/feature_selection/test_feature_selection.py b/feature_selection/test_feature_selection.py index 6289648f2..d21e648ff 100644 --- a/feature_selection/test_feature_selection.py +++ b/feature_selection/test_feature_selection.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from mlrun import code_to_function +from mlrun import code_to_function, get_dataitem from pathlib import Path import shutil @@ -44,5 +44,5 @@ def test_run_local_feature_selection(): inputs={'df_artifact': 'data/metrics.pq'}, artifact_path='artifacts/', ) - assert run.artifact('feature_scores').get() and run.artifact('selected_features').get() + assert run.outputs['feature_scores'] and run.outputs['selected_features'] _delete_outputs({ARTIFACTS_PATH, RUNS_PATH, SCHEDULES_PATH}) From 7f3403ced7d291e52305b272c185f7506c84cdca Mon Sep 17 00:00:00 2001 From: Avi Asulin <34214569+aviaIguazio@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:42:06 +0300 Subject: [PATCH 7/8] Update feature_selection/test_feature_selection.py Co-authored-by: Eyal Danieli --- feature_selection/test_feature_selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_selection/test_feature_selection.py b/feature_selection/test_feature_selection.py index d21e648ff..9cb5ca621 100644 --- a/feature_selection/test_feature_selection.py +++ b/feature_selection/test_feature_selection.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from mlrun import code_to_function, get_dataitem +from mlrun import code_to_function from pathlib import Path import shutil From 75dbafd698c6cfe41aa59b9663b22c3ea2a2b64f Mon Sep 17 00:00:00 2001 From: Avi Asulin <34214569+aviaIguazio@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:43:08 +0300 Subject: [PATCH 8/8] Update item.yaml --- feature_selection/item.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_selection/item.yaml b/feature_selection/item.yaml index 1b25ec410..99675b4e8 100644 --- a/feature_selection/item.yaml +++ b/feature_selection/item.yaml @@ -22,4 +22,4 @@ spec: kind: job requirements: [] url: '' -version: 1.7.0 +version: 1.5.0