diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000..ceef8a5
Binary files /dev/null and b/.DS_Store differ
diff --git a/src/cls/CacheUpdater/Task.cls b/src/cls/CacheUpdater/Task.cls
new file mode 100755
index 0000000..08c6c50
--- /dev/null
+++ b/src/cls/CacheUpdater/Task.cls
@@ -0,0 +1,245 @@
+Class CacheUpdater.Task Extends (%SYS.Task.Definition, CacheUpdater.UDL)
+{
+
+Parameter TaskName = "GitHub Update";
+
+/// Repository URL, like https://github.com/intersystems-ru/Cache-MDX2JSON
+/// Increased to 500 to support long urls
+Property GitHubURL As %String(MAXLEN = 500);
+
+/// GitHub user, who has access to repository. Optional for public repositories.
+/// Note, that with Username/Password, you can make up to 5,000 requests per hour.
+/// For unauthenticated requests, the rate limit allows to make up to 60 requests per hour.
+/// Unauthenticated requests are associated with an IP address.
+/// Required, if you want to create webhooks
+Property Username As %String;
+
+/// GitHub password, corresponding to Username. Optional for public repositories.
+Property Password As %String;
+
+/// Namespace, where to download and compile repository
+Property Namespace As %String [ InitialExpression = {$Namespace} ];
+
+/// Repository branch, usually master. Leave empty, if you want to receive default branch.
+Property Branch As %String [ InitialExpression = "master" ];
+
+Method OnTask() As %Status
+{
+ Return:'##class(%SYS.Namespace).Exists(..Namespace) $$$ERROR($$$NamespaceUnavailable,..Namespace)
+
+ Set Owner = $p(..GitHubURL,"/",4)
+ Set Repository = $p(..GitHubURL,"/",5)
+
+ Return ..Update(Owner, Repository, ..Branch, ..Username, ..Password, ..Namespace)
+}
+
+/// Downloads and compiles GitHub repository.
+/// Owner - The name of the repository owner.
+/// Repository - The name of the repository.
+/// Branch - The name of the commit/branch/tag. If skipped the repository’s default branch (usually master) would be used.
+/// Username - GitHub user, who has access to repository. Optional for public repositories.
+/// Password - GitHub password, corresponding to Username. Optional for public repositories.
+/// Note, that with Username, you can make up to 5,000 requests per hour.
+/// For unauthenticated requests, the rate limit allows to make up to 60 requests per hour.
+/// Unauthenticated requests are associated with an IP address.
+/// Namespace - Namespace, where to download and compile repository.
+///
+/// For example in the repository: https://github.com/intersystems-ru/Cache-MDX2JSON
+/// Owner - intersystems-ru, Repository - Cache-MDX2JSON.
+ClassMethod Update(Owner As %String, Repository As %String, Branch As %String = "", Username As %String = "", Password As %String = "", Namespace = {$Namespace}) As %Status
+{
+ #dim req As %Net.HttpRequest
+ Set req = ..CreateRequest(Username, Password)
+ Set req.Location = "repos/" _ Owner _ "/" _ Repository _ "/contents" // as described in https://developer.github.com/v3/repos/
+
+ Set links = ##class(%ListOfDataTypes).%New()
+ Set st = ..ProcessDirectory("",.req,Branch,.links)
+ Return:$$$ISERR(st) st
+
+ Set namespace = $Namespace
+ Zn Namespace
+ Set st = ..DownloadFiles(links,req,.list)
+ zw list
+ Set st2 = $system.OBJ.CompileList(.list,"cuk /checkuptodate=expandedonly")
+ Zn namespace
+
+ Return $$$ADDSC(st, st2)
+}
+
+/// Process one directory of GitHub repository. Recursive.
+/// Path -Internal repository path. Root is empty string
+/// Request - Authenticated/Set %Net.HttpRequest object.
+/// Links - List of links to raw files (which satisfy IsCacheFile conditions) from repository.
+ClassMethod ProcessDirectory(Path As %String = "", Request As %Net.HttpRequest, Branch As %String = "", ByRef Links As %ListOfDataTypes) As %Status
+{
+ Set location = Request.Location
+ Set Request.Location = Request.Location _ Path
+ Do:(Branch'="") Request.SetParam("ref",Branch)
+
+ Set st = Request.Get()
+
+ Return:$$$ISERR(st) st
+ Return:(Request.HttpResponse.StatusCode = 404) $$$ERROR($$$GeneralError,"Repository doesn't exist OR you don't have access")
+ Return:((Request.HttpResponse.StatusCode = 403) && (Request.HttpResponse.GetHeader("X-RATELIMIT-REMAINING")=0)) $$$ERROR($$$GeneralError,"API rate limit exceeded. Try logging in.")
+ Return:(Request.HttpResponse.StatusCode '= 200) $$$ERROR($$$GeneralError,"Received " _ Request.HttpResponse.StatusCode _ " expected 200")
+
+ #dim objects As List of %ZEN.proxyObject
+ #dim obj As %ZEN.proxyObject
+ Set st = ##class(%ZEN.Auxiliary.jsonProvider).%ConvertJSONToObject(Request.HttpResponse.Data,,.objects,1)
+ Return:$$$ISERR(st) st
+
+ For i = 1:1:objects.Count() {
+ Set obj = objects.GetAt(i)
+ If (obj.type = "dir") {
+ Set st = ..ProcessDirectory("/"_obj.name,Request,Branch,.Links)
+ Return:$$$ISERR(st) st
+ } ElseIf (obj.type = "file") {
+ //Do:..IsCacheFile(obj) Links.Insert(obj."download_url")
+ Do Links.Insert($LB(obj."download_url",..IsCacheFile(obj)))
+ } Else {
+ // obj.type = "symlink" or obj.type = "submodule"
+ }
+ }
+ Set Request.Location = location // to keep track of where in the repository tree we are
+ Return $$$OK
+}
+
+/// Check that incoming file is the one you need.
+ClassMethod IsCacheFile(File As %ZEN.proxyObject) As %Boolean
+{
+ Set extensions = ",xml,cls,csp,csr,mac,int,bas,inc,gbl,prj,obj,pkg,gof,dfi,pivot,dashboard,html,css,js,ts,scss,"
+ Return:($L(File.name,".")=1) 0 //no extension
+ Set File.Extension = $P(File.name,".",$L(File.name,"."))
+ Return $F(extensions,","_$ZCVT(File.Extension,"l")_",")
+}
+
+/// Download list of files on https://raw.githubusercontent.com/ server.
+/// Links - List of links to raw files.
+/// Request - Authenticated/Set %Net.HttpRequest object.
+/// loadedlist - Returns an array of the items loaded.
+ClassMethod DownloadFiles(Links As %ListOfDataTypes, Request As %Net.HttpRequest, Output Items) As %Status
+{
+ Kill Items
+ Set Request.Server = "raw.githubusercontent.com"
+ Set st = $$$OK
+ Try
+ {
+ For i = 1:1:Links.Count()
+ {
+ Set link = $ListGet(Links.GetAt(i),1)
+ Set bIsCacheFile = $ListGet(Links.GetAt(i),2)
+ Set ^gitfiles(i,"link")=link
+ Set ^gitfiles(i,"bIsCacheFile")=bIsCacheFile
+
+ Set streq = Request.Get($e(link,35,*)) // Remove "https://raw.githubusercontent.com/" from URL.
+ If $$$ISERR(streq)
+ {
+ Set st=$$$ADDSC(st, streq)
+ Set ^gitfiles(i,"streq")=streq
+ Continue
+ }
+
+ Set ^gitfiles(i,"stream")="starting..."
+ Set binarystream = Request.HttpResponse.Data
+
+ Do binarystream.Rewind() // just in case
+
+ Set characterStream=##class(%GlobalCharacterStream).%New() //translating binary stream into character stream
+ Set stTranslate=$$$OK
+ Try
+ {
+ While 'binarystream.AtEnd
+ {
+ //Use eol to prevent breaking lines larger than 32Kb
+ Set line=binarystream.ReadLine(, .stTranslate, .eol)
+ Quit:$System.Status.IsError(stTranslate)
+
+ If eol
+ {
+ Set stTranslate=characterStream.WriteLine(line)
+ }
+ Else
+ {
+ Set stTranslate=characterStream.Write(line)
+ }
+ Quit:$System.Status.IsError(stTranslate)
+ }
+ Quit:$System.Status.IsError(stTranslate)
+
+ Do characterStream.Rewind()
+ }
+ Catch (oTranslateStreamException)
+ {
+ Set stTranslate=oTranslateStreamException.AsStatus()
+ }
+
+ If $System.Status.IsError(stTranslate)
+ {
+ //Could not convert binary stream to character stream
+ //It is probably a binary file anyway
+ Set characterStream=""
+ Set st=$$$ADDSC(st, stTranslate)
+ Set ^gitfiles(i,"stTranslate")=stTranslate
+ }
+ Set ^gitfiles(i,"stream")="Done"
+
+ Do binarystream.Rewind()
+
+ Set stload = $$$OK
+
+ set items = ""
+ If ('$IsObject(characterStream)) || (..IsUDLFile(characterStream))
+ {
+ Set ^gitfiles(i,"IsUDLFile")="1"
+ Set stload = ..LoadUDLFile(characterStream, binarystream, link, .items)
+ }
+ ElseIf bIsCacheFile
+ {
+ Set ^gitfiles(i,"IsUDLFile")="0"
+ Set stload = $system.OBJ.LoadStream(characterStream,"",.error,.items,,,,"UTF8")
+ }
+ Set ^gitfiles(i,"stload")=stload
+ If $$$ISERR(stload)
+ {
+ Set st=$$$ADDSC(st, stload)
+ Continue
+ }
+ Merge Items = items // Does not overwrite existing array keys: Items(itemname)=""
+ }
+
+ Set Request.Server="api.github.com"
+ }
+ Catch (oException)
+ {
+ Set st = oException.AsStatus()
+ If $D(i) Set ^gitfiles(i,"st final")=st
+ }
+
+ Quit st
+}
+
+ClassMethod CreateRequest(Username As %String, Password As %String) As %Net.HttpRequest
+{
+ Set namespace = $Namespace
+ Set SSLConfig = "GitHub"
+
+ Zn "%SYS"
+ Do:'##class(Security.SSLConfigs).Exists(SSLConfig) ##class(Security.SSLConfigs).Create(SSLConfig)
+ Zn namespace
+
+ Set req=##class(%Net.HttpRequest).%New()
+ Set req.Https=1
+ Set req.SSLConfiguration=SSLConfig
+ Set req.Server="api.github.com"
+ Do req.SetHeader("Accept","application/vnd.github.v3+json") // we want 3rd version of api
+
+ If ($d(Username) && $d(Password) && (Username'="") && (Password'="")) { // supply Username and Password, if both are provided. GitHub accept Basic Auth
+ Set req.Username = Username // https://developer.github.com/v3/auth/
+ Set req.Password = Password
+ }
+
+ Return req
+}
+
+}
+
diff --git a/src/cls/CacheUpdater/UDL.cls b/src/cls/CacheUpdater/UDL.cls
new file mode 100755
index 0000000..ee973e3
--- /dev/null
+++ b/src/cls/CacheUpdater/UDL.cls
@@ -0,0 +1,337 @@
+Class CacheUpdater.UDL Extends %RegisteredObject
+{
+
+/// Checks whether this file is in UDL format
+/// stream - stream which contains file definition
+ClassMethod IsUDLFile(stream As %GlobalCharacterStream) As %Boolean
+{
+ // probably 10 lines is enough
+ set counter = 0
+ while 'stream.AtEnd {
+ if counter >= 10 {
+ quit
+ }
+ set line = stream.ReadLine()
+ if $find(line, "line - any string.
+ClassMethod ReadName(line As %String) As %String
+{
+ set trimmed = $zstrip(line, "<>W")
+ return $piece(trimmed, " ")
+}
+
+/// Finds a name of a class
+/// stream - stream which contains a class definition
+/// name - name which contains the name of class
+ClassMethod GetClassName(stream As %GlobalCharacterStream, ByRef name As %String) As %Status
+{
+ while 'stream.AtEnd {
+ set line = stream.ReadLine()
+
+ if $extract(line, 1, 3) = "///" { // check for inline comments
+ continue
+ } elseif $zconvert($extract(line, 1, 5), "l") = "class" {
+ set line = $extract(line, 6, *)
+ set name = ..ReadName(line)
+ if name = "" {
+ return '$$$OK
+ } else {
+ return $$$OK
+ }
+ }
+ }
+ return '$$$OK
+}
+
+/// Finds a name of a routine
+/// stream - stream which contains a routine definition
+/// name - name which contains the name of routine
+/// type - type of file {1 - mac, 2 - inc, 3 - int}
+ClassMethod GetRoutineName(stream As %GlobalCharacterStream, ByRef name As %String, ByRef type As %Integer) As %Status
+{
+ while 'stream.AtEnd {
+ set line = stream.ReadLine()
+ set index = $find(line, "ROUTINE")
+ // TODO - check whether the name on the next line
+ // or something is between ROUTINE and name
+ if index {
+ if $find(line, "[Type=INC]") {
+ set type = 2
+ }
+ elseif $find(line, "[Type=INT,Generated]") {
+ set type = 3
+ }
+ else {
+ set type = 1
+ }
+ set line = $extract(line, index, *)
+ set name = ..ReadName(line)
+ if name = "" {
+ return '$$$OK
+ } else {
+ return $$$OK
+ }
+ }
+ }
+ return '$$$OK
+}
+
+/// Finds a name of a dfi
+/// stream - stream which contains a dfi definition
+/// name - name which contains the name of dfi
+ClassMethod GetDFIName(stream As %GlobalCharacterStream, ByRef name As %String) As %Status
+{
+ #dim textreader As %XML.TextReader
+ set dfiContent = ""
+
+ // I don't know why but if i just parse stream it doesn't work
+ while 'stream.AtEnd {
+ set dfiContent = dfiContent _ stream.Read()
+ }
+
+ set st = ##class(%XML.TextReader).ParseString(dfiContent, .textreader)
+ return:$$$ISERR(st) st
+
+ while textreader.Read() {
+ set node = textreader.Name
+ if (node = "pivot") || (node = "dashboard") {
+ do textreader.MoveToAttributeName("folderName")
+ // set dfiFolderName = $translate(textreader.Value, " ", "-")
+ set dfiFolderName=textreader.Value
+
+ do textreader.MoveToAttributeName("name")
+ // set dfiName = $translate(textreader.Value, " ", "-")
+ set dfiName=textreader.Value
+ set name = dfiFolderName _ "-" _ dfiName _ "." _ node _ ".dfi"
+ return $$$OK
+ }
+ }
+ return '$$$OK
+}
+
+/// Get extension of the file by url
+/// url - the url where the file is located in the web.
+ClassMethod GetExt(url As %String) As %String
+{
+ //return $zconvert($piece(url, ".", *), "l")
+ //AMIR: There are parameters after the extension that are not part of the extension
+ return $zconvert($piece($piece(url, ".", *),"?"), "l")
+}
+
+/// Check whether a file is a web file
+/// ext - extensions of the file
+ClassMethod IsWebFile(ext As %String) As %String
+{
+ set webExts = "csp,html,css,js,ts,scss"
+ return $find(webExts, ext)
+}
+
+/// Imports the file in UDL file in the project
+/// contentStream - the stream which contains the source code in udl format.
+/// url - the url where the file is located in the web.
+/// list - array of files to compile
+ClassMethod LoadUDLFile(contentStream As %GlobalCharacterStream, binaryStream As %Stream.FileCharacterGzip, url As %String, list As %String) As %Status
+{
+ set st = $$$OK
+
+ set ext = ..GetExt(url)
+
+ if ext = "cls" {
+ set st = ..CreateClass(contentStream, url, .list)
+ }
+ elseif ext = "dfi" {
+ set st = ..CreateDFI(contentStream, url, .list)
+ }
+ elseif (ext = "inc") || (ext = "mac") {
+ set st = ..CreateRoutine(contentStream, url, .list)
+ }
+ else
+ {
+ set st = ..CreateWebFile(contentStream, binaryStream, url, ext, .list)
+ }
+ return st
+}
+
+/// Checks whether the class exists
+/// className - name of the class.
+ClassMethod DoesClassExist(className As %String) As %Boolean
+{
+ Set query = "SELECT TOP 1 COUNT(ID) FROM %Dictionary.ClassDefinition WHERE ID = ?"
+ Set statement = ##class(%SQL.Statement).%New()
+ Set st = statement.%Prepare(query)
+ Set rset = statement.%Execute(className)
+ If (rset.%Next()) && (rset.%ROWCOUNT > 0) {
+ Return $$$YES
+ }
+ Return $$$NO
+}
+
+/// Creates and imports the class into the project from stream
+/// contentStream - the stream which contains the source code in udl format.
+/// url - the url where the file is located in the web.
+/// list - array of files to compile
+ClassMethod CreateClass(contentStream As %CharacterStream, url As %String, ByRef list As %String) As %Status
+{
+ Set st = ..GetClassName(contentStream, .className)
+ Return:$$$ISERR(st) st
+
+ set list(className _ ".cls") = ""
+
+ Do contentStream.Rewind()
+
+ if '(##class(%Dictionary.ClassDefinition).%ExistsId(className)) {
+ Set clsDef = ##class(%Dictionary.ClassDefinition).%New()
+ Set clsDef.Name = className
+ Set st = clsDef.%Save()
+ Return:$$$ISERR(st) st
+ }
+
+
+ Set namespace = $namespace
+ Set st = ##class(%Compiler.UDL.TextServices).SetTextFromStream(namespace, className, contentStream)
+
+ if st {
+ w !, "Imported " _ className, !
+ }
+
+ Return st
+}
+
+/// Creates and imports the dfi file into the project from stream
+/// contentStream - the stream which contains the source code in udl format.
+/// list - array of files to compile
+ClassMethod CreateDFI(contentStream As %CharacterStream, url As %String, ByRef list As %String) As %Status
+{
+ Set st = $$$OK
+ Try {
+ Set st = ..GetDFIName(contentStream, .name)
+ Return:$$$ISERR(st) st
+
+ set list(name) = ""
+
+ Set tDoc = ##class(%DeepSee.UI.FolderItemDocument).%New(name)
+ Set st = tDoc.ImportFromXML(contentStream)
+ Return:$$$ISERR(st) st
+
+ Set st = tDoc.Save()
+ if st {
+ w !, "Imported " _ name, !
+ }
+ Return:$$$ISERR(st) st
+ } Catch e {
+ Set st = e.AsStatus()
+ }
+ Return st
+}
+
+/// Creates and imports mac, int, inc files into the project from stream
+/// contentStream - the stream which contains the source code in udl format.
+/// url - the url where the file is located in the web.
+/// list - array of files to compile
+ClassMethod CreateRoutine(contentStream As %GlobalCharacterStream, url As %String, ByRef list As %String) As %Status
+{
+ Set st = ..GetRoutineName(contentStream, .name, .type)
+ do contentStream.Rewind()
+
+ return:$$$ISERR(st) st
+
+ if type = 1 {
+ set name = name _ ".mac"
+ }
+ elseif type = 2 {
+ set name = name _ ".inc"
+ }
+ elseif type = 3 {
+ set name = name _ ".int"
+ }
+
+ set list(name) = ""
+
+ Set rtn = ##class(%Routine).%New(name)
+ While 'contentStream.AtEnd {
+ Set line = contentStream.ReadLine()
+ If $Find(line, "ROUTINE") {
+ Continue
+ }
+ Do rtn.WriteLine(line)
+ }
+
+ Set st = rtn.Save()
+ Return:$$$ISERR(st) st
+
+ if st {
+ w !, "Imported " _ name, !
+ }
+ Return st
+}
+
+/// Creates and imports mac, int, inc files into the project from stream
+/// contentStream - the stream which contains the source code in udl format.
+/// url - the url where the file is located in the web.
+/// ext - extension of the file
+/// list - array of files to compile
+ClassMethod CreateWebFile(contentStream As %GlobalCharacterStream, binaryStream As %Stream.FileCharacterGzip, url As %String, ext As %String, ByRef list As %String) As %Status
+{
+ Set st = $$$OK
+ Try
+ {
+ Set tDefaultApp=$system.CSP.GetDefaultApp($namespace)_"/"
+ set tCSPRootPath = $system.CSP.GetFileName(tDefaultApp)
+
+ Set tFileName = $Piece($Piece(url,"?",1),"/",*)
+
+ Set tCSPSubPath = $Piece(url,tDefaultApp,2,*) //still has filename and ? parameters in it
+ Set tCSPSubPath = $Piece(tCSPSubPath, tFileName,1) //now it does't
+
+ set tFileDirectory = tCSPRootPath_tCSPSubPath
+ Set tFullFileName = tFileDirectory_tFileName
+
+ //On Windows, tFullFileName will contain \ and / but CreateDirectoryChain() and
+ //LinkToFile() already normalize the paths accordingly to the OS for us so
+ //we don't have to worry about it.
+ If '##class(%File).CreateDirectoryChain(tFileDirectory)
+ {
+ Set st = $System.Status.Error(5001,"Could nor create path chain '"_tFileDirectory_"'")
+ Quit
+ }
+
+ Set filestream = ##class(%Stream.FileCharacter).%New()
+ set st = filestream.LinkToFile(tFullFileName)
+ Quit:$System.Status.IsError(st)
+
+ If $IsObject(contentStream) && ..IsWebFile(ext)
+ {
+ Set st=filestream.CopyFrom(contentStream)
+ }
+ Else
+ {
+ Set st=filestream.CopyFrom(binaryStream)
+ }
+ Quit:$System.Status.IsError(st)
+
+ set st = filestream.%Save()
+ Quit:$System.Status.IsError(st)
+
+ Write !, "Imported " _ tFullFileName, !
+ }
+ Catch (oException)
+ {
+ Set st = oException.AsStatus()
+ }
+
+ Quit st
+}
+
+}
+