diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..ceef8a5 Binary files /dev/null and b/.DS_Store differ diff --git a/src/cls/CacheUpdater/Task.cls b/src/cls/CacheUpdater/Task.cls new file mode 100755 index 0000000..08c6c50 --- /dev/null +++ b/src/cls/CacheUpdater/Task.cls @@ -0,0 +1,245 @@ +Class CacheUpdater.Task Extends (%SYS.Task.Definition, CacheUpdater.UDL) +{ + +Parameter TaskName = "GitHub Update"; + +/// Repository URL, like https://github.com/intersystems-ru/Cache-MDX2JSON +/// Increased to 500 to support long urls +Property GitHubURL As %String(MAXLEN = 500); + +/// GitHub user, who has access to repository. Optional for public repositories.
+/// Note, that with Username/Password, you can make up to 5,000 requests per hour. +/// For unauthenticated requests, the rate limit allows to make up to 60 requests per hour. +/// Unauthenticated requests are associated with an IP address.
+/// Required, if you want to create webhooks +Property Username As %String; + +/// GitHub password, corresponding to Username. Optional for public repositories. +Property Password As %String; + +/// Namespace, where to download and compile repository +Property Namespace As %String [ InitialExpression = {$Namespace} ]; + +/// Repository branch, usually master. Leave empty, if you want to receive default branch. +Property Branch As %String [ InitialExpression = "master" ]; + +Method OnTask() As %Status +{ + Return:'##class(%SYS.Namespace).Exists(..Namespace) $$$ERROR($$$NamespaceUnavailable,..Namespace) + + Set Owner = $p(..GitHubURL,"/",4) + Set Repository = $p(..GitHubURL,"/",5) + + Return ..Update(Owner, Repository, ..Branch, ..Username, ..Password, ..Namespace) +} + +/// Downloads and compiles GitHub repository.
+/// Owner - The name of the repository owner.
+/// Repository - The name of the repository.
+/// Branch - The name of the commit/branch/tag. If skipped the repository’s default branch (usually master) would be used.
+/// Username - GitHub user, who has access to repository. Optional for public repositories.
+/// Password - GitHub password, corresponding to Username. Optional for public repositories.
+/// Note, that with Username, you can make up to 5,000 requests per hour. +/// For unauthenticated requests, the rate limit allows to make up to 60 requests per hour. +/// Unauthenticated requests are associated with an IP address.
+/// Namespace - Namespace, where to download and compile repository.
+/// +/// For example in the repository: https://github.com/intersystems-ru/Cache-MDX2JSON
+/// Owner - intersystems-ru, Repository - Cache-MDX2JSON.
+ClassMethod Update(Owner As %String, Repository As %String, Branch As %String = "", Username As %String = "", Password As %String = "", Namespace = {$Namespace}) As %Status +{ + #dim req As %Net.HttpRequest + Set req = ..CreateRequest(Username, Password) + Set req.Location = "repos/" _ Owner _ "/" _ Repository _ "/contents" // as described in https://developer.github.com/v3/repos/ + + Set links = ##class(%ListOfDataTypes).%New() + Set st = ..ProcessDirectory("",.req,Branch,.links) + Return:$$$ISERR(st) st + + Set namespace = $Namespace + Zn Namespace + Set st = ..DownloadFiles(links,req,.list) + zw list + Set st2 = $system.OBJ.CompileList(.list,"cuk /checkuptodate=expandedonly") + Zn namespace + + Return $$$ADDSC(st, st2) +} + +/// Process one directory of GitHub repository. Recursive.
+/// Path -Internal repository path. Root is empty string
+/// Request - Authenticated/Set %Net.HttpRequest object.
+/// Links - List of links to raw files (which satisfy IsCacheFile conditions) from repository.
+ClassMethod ProcessDirectory(Path As %String = "", Request As %Net.HttpRequest, Branch As %String = "", ByRef Links As %ListOfDataTypes) As %Status +{ + Set location = Request.Location + Set Request.Location = Request.Location _ Path + Do:(Branch'="") Request.SetParam("ref",Branch) + + Set st = Request.Get() + + Return:$$$ISERR(st) st + Return:(Request.HttpResponse.StatusCode = 404) $$$ERROR($$$GeneralError,"Repository doesn't exist OR you don't have access") + Return:((Request.HttpResponse.StatusCode = 403) && (Request.HttpResponse.GetHeader("X-RATELIMIT-REMAINING")=0)) $$$ERROR($$$GeneralError,"API rate limit exceeded. Try logging in.") + Return:(Request.HttpResponse.StatusCode '= 200) $$$ERROR($$$GeneralError,"Received " _ Request.HttpResponse.StatusCode _ " expected 200") + + #dim objects As List of %ZEN.proxyObject + #dim obj As %ZEN.proxyObject + Set st = ##class(%ZEN.Auxiliary.jsonProvider).%ConvertJSONToObject(Request.HttpResponse.Data,,.objects,1) + Return:$$$ISERR(st) st + + For i = 1:1:objects.Count() { + Set obj = objects.GetAt(i) + If (obj.type = "dir") { + Set st = ..ProcessDirectory("/"_obj.name,Request,Branch,.Links) + Return:$$$ISERR(st) st + } ElseIf (obj.type = "file") { + //Do:..IsCacheFile(obj) Links.Insert(obj."download_url") + Do Links.Insert($LB(obj."download_url",..IsCacheFile(obj))) + } Else { + // obj.type = "symlink" or obj.type = "submodule" + } + } + Set Request.Location = location // to keep track of where in the repository tree we are + Return $$$OK +} + +/// Check that incoming file is the one you need. +ClassMethod IsCacheFile(File As %ZEN.proxyObject) As %Boolean +{ + Set extensions = ",xml,cls,csp,csr,mac,int,bas,inc,gbl,prj,obj,pkg,gof,dfi,pivot,dashboard,html,css,js,ts,scss," + Return:($L(File.name,".")=1) 0 //no extension + Set File.Extension = $P(File.name,".",$L(File.name,".")) + Return $F(extensions,","_$ZCVT(File.Extension,"l")_",") +} + +/// Download list of files on https://raw.githubusercontent.com/ server.
+/// Links - List of links to raw files.
+/// Request - Authenticated/Set %Net.HttpRequest object.
+/// loadedlist - Returns an array of the items loaded. +ClassMethod DownloadFiles(Links As %ListOfDataTypes, Request As %Net.HttpRequest, Output Items) As %Status +{ + Kill Items + Set Request.Server = "raw.githubusercontent.com" + Set st = $$$OK + Try + { + For i = 1:1:Links.Count() + { + Set link = $ListGet(Links.GetAt(i),1) + Set bIsCacheFile = $ListGet(Links.GetAt(i),2) + Set ^gitfiles(i,"link")=link + Set ^gitfiles(i,"bIsCacheFile")=bIsCacheFile + + Set streq = Request.Get($e(link,35,*)) // Remove "https://raw.githubusercontent.com/" from URL. + If $$$ISERR(streq) + { + Set st=$$$ADDSC(st, streq) + Set ^gitfiles(i,"streq")=streq + Continue + } + + Set ^gitfiles(i,"stream")="starting..." + Set binarystream = Request.HttpResponse.Data + + Do binarystream.Rewind() // just in case + + Set characterStream=##class(%GlobalCharacterStream).%New() //translating binary stream into character stream + Set stTranslate=$$$OK + Try + { + While 'binarystream.AtEnd + { + //Use eol to prevent breaking lines larger than 32Kb + Set line=binarystream.ReadLine(, .stTranslate, .eol) + Quit:$System.Status.IsError(stTranslate) + + If eol + { + Set stTranslate=characterStream.WriteLine(line) + } + Else + { + Set stTranslate=characterStream.Write(line) + } + Quit:$System.Status.IsError(stTranslate) + } + Quit:$System.Status.IsError(stTranslate) + + Do characterStream.Rewind() + } + Catch (oTranslateStreamException) + { + Set stTranslate=oTranslateStreamException.AsStatus() + } + + If $System.Status.IsError(stTranslate) + { + //Could not convert binary stream to character stream + //It is probably a binary file anyway + Set characterStream="" + Set st=$$$ADDSC(st, stTranslate) + Set ^gitfiles(i,"stTranslate")=stTranslate + } + Set ^gitfiles(i,"stream")="Done" + + Do binarystream.Rewind() + + Set stload = $$$OK + + set items = "" + If ('$IsObject(characterStream)) || (..IsUDLFile(characterStream)) + { + Set ^gitfiles(i,"IsUDLFile")="1" + Set stload = ..LoadUDLFile(characterStream, binarystream, link, .items) + } + ElseIf bIsCacheFile + { + Set ^gitfiles(i,"IsUDLFile")="0" + Set stload = $system.OBJ.LoadStream(characterStream,"",.error,.items,,,,"UTF8") + } + Set ^gitfiles(i,"stload")=stload + If $$$ISERR(stload) + { + Set st=$$$ADDSC(st, stload) + Continue + } + Merge Items = items // Does not overwrite existing array keys: Items(itemname)="" + } + + Set Request.Server="api.github.com" + } + Catch (oException) + { + Set st = oException.AsStatus() + If $D(i) Set ^gitfiles(i,"st final")=st + } + + Quit st +} + +ClassMethod CreateRequest(Username As %String, Password As %String) As %Net.HttpRequest +{ + Set namespace = $Namespace + Set SSLConfig = "GitHub" + + Zn "%SYS" + Do:'##class(Security.SSLConfigs).Exists(SSLConfig) ##class(Security.SSLConfigs).Create(SSLConfig) + Zn namespace + + Set req=##class(%Net.HttpRequest).%New() + Set req.Https=1 + Set req.SSLConfiguration=SSLConfig + Set req.Server="api.github.com" + Do req.SetHeader("Accept","application/vnd.github.v3+json") // we want 3rd version of api + + If ($d(Username) && $d(Password) && (Username'="") && (Password'="")) { // supply Username and Password, if both are provided. GitHub accept Basic Auth + Set req.Username = Username // https://developer.github.com/v3/auth/ + Set req.Password = Password + } + + Return req +} + +} + diff --git a/src/cls/CacheUpdater/UDL.cls b/src/cls/CacheUpdater/UDL.cls new file mode 100755 index 0000000..ee973e3 --- /dev/null +++ b/src/cls/CacheUpdater/UDL.cls @@ -0,0 +1,337 @@ +Class CacheUpdater.UDL Extends %RegisteredObject +{ + +/// Checks whether this file is in UDL format +/// stream - stream which contains file definition +ClassMethod IsUDLFile(stream As %GlobalCharacterStream) As %Boolean +{ + // probably 10 lines is enough + set counter = 0 + while 'stream.AtEnd { + if counter >= 10 { + quit + } + set line = stream.ReadLine() + if $find(line, "line - any string.
+ClassMethod ReadName(line As %String) As %String +{ + set trimmed = $zstrip(line, "<>W") + return $piece(trimmed, " ") +} + +/// Finds a name of a class +/// stream - stream which contains a class definition
+/// name - name which contains the name of class
+ClassMethod GetClassName(stream As %GlobalCharacterStream, ByRef name As %String) As %Status +{ + while 'stream.AtEnd { + set line = stream.ReadLine() + + if $extract(line, 1, 3) = "///" { // check for inline comments + continue + } elseif $zconvert($extract(line, 1, 5), "l") = "class" { + set line = $extract(line, 6, *) + set name = ..ReadName(line) + if name = "" { + return '$$$OK + } else { + return $$$OK + } + } + } + return '$$$OK +} + +/// Finds a name of a routine +/// stream - stream which contains a routine definition
+/// name - name which contains the name of routine
+/// type - type of file {1 - mac, 2 - inc, 3 - int} +ClassMethod GetRoutineName(stream As %GlobalCharacterStream, ByRef name As %String, ByRef type As %Integer) As %Status +{ + while 'stream.AtEnd { + set line = stream.ReadLine() + set index = $find(line, "ROUTINE") + // TODO - check whether the name on the next line + // or something is between ROUTINE and name + if index { + if $find(line, "[Type=INC]") { + set type = 2 + } + elseif $find(line, "[Type=INT,Generated]") { + set type = 3 + } + else { + set type = 1 + } + set line = $extract(line, index, *) + set name = ..ReadName(line) + if name = "" { + return '$$$OK + } else { + return $$$OK + } + } + } + return '$$$OK +} + +/// Finds a name of a dfi +/// stream - stream which contains a dfi definition
+/// name - name which contains the name of dfi
+ClassMethod GetDFIName(stream As %GlobalCharacterStream, ByRef name As %String) As %Status +{ + #dim textreader As %XML.TextReader + set dfiContent = "" + + // I don't know why but if i just parse stream it doesn't work + while 'stream.AtEnd { + set dfiContent = dfiContent _ stream.Read() + } + + set st = ##class(%XML.TextReader).ParseString(dfiContent, .textreader) + return:$$$ISERR(st) st + + while textreader.Read() { + set node = textreader.Name + if (node = "pivot") || (node = "dashboard") { + do textreader.MoveToAttributeName("folderName") + // set dfiFolderName = $translate(textreader.Value, " ", "-") + set dfiFolderName=textreader.Value + + do textreader.MoveToAttributeName("name") + // set dfiName = $translate(textreader.Value, " ", "-") + set dfiName=textreader.Value + set name = dfiFolderName _ "-" _ dfiName _ "." _ node _ ".dfi" + return $$$OK + } + } + return '$$$OK +} + +/// Get extension of the file by url +/// url - the url where the file is located in the web.
+ClassMethod GetExt(url As %String) As %String +{ + //return $zconvert($piece(url, ".", *), "l") + //AMIR: There are parameters after the extension that are not part of the extension + return $zconvert($piece($piece(url, ".", *),"?"), "l") +} + +/// Check whether a file is a web file +/// ext - extensions of the file
+ClassMethod IsWebFile(ext As %String) As %String +{ + set webExts = "csp,html,css,js,ts,scss" + return $find(webExts, ext) +} + +/// Imports the file in UDL file in the project +/// contentStream - the stream which contains the source code in udl format.
+/// url - the url where the file is located in the web.
+/// list - array of files to compile
+ClassMethod LoadUDLFile(contentStream As %GlobalCharacterStream, binaryStream As %Stream.FileCharacterGzip, url As %String, list As %String) As %Status +{ + set st = $$$OK + + set ext = ..GetExt(url) + + if ext = "cls" { + set st = ..CreateClass(contentStream, url, .list) + } + elseif ext = "dfi" { + set st = ..CreateDFI(contentStream, url, .list) + } + elseif (ext = "inc") || (ext = "mac") { + set st = ..CreateRoutine(contentStream, url, .list) + } + else + { + set st = ..CreateWebFile(contentStream, binaryStream, url, ext, .list) + } + return st +} + +/// Checks whether the class exists +/// className - name of the class.
+ClassMethod DoesClassExist(className As %String) As %Boolean +{ + Set query = "SELECT TOP 1 COUNT(ID) FROM %Dictionary.ClassDefinition WHERE ID = ?" + Set statement = ##class(%SQL.Statement).%New() + Set st = statement.%Prepare(query) + Set rset = statement.%Execute(className) + If (rset.%Next()) && (rset.%ROWCOUNT > 0) { + Return $$$YES + } + Return $$$NO +} + +/// Creates and imports the class into the project from stream +/// contentStream - the stream which contains the source code in udl format.
+/// url - the url where the file is located in the web.
+/// list - array of files to compile
+ClassMethod CreateClass(contentStream As %CharacterStream, url As %String, ByRef list As %String) As %Status +{ + Set st = ..GetClassName(contentStream, .className) + Return:$$$ISERR(st) st + + set list(className _ ".cls") = "" + + Do contentStream.Rewind() + + if '(##class(%Dictionary.ClassDefinition).%ExistsId(className)) { + Set clsDef = ##class(%Dictionary.ClassDefinition).%New() + Set clsDef.Name = className + Set st = clsDef.%Save() + Return:$$$ISERR(st) st + } + + + Set namespace = $namespace + Set st = ##class(%Compiler.UDL.TextServices).SetTextFromStream(namespace, className, contentStream) + + if st { + w !, "Imported " _ className, ! + } + + Return st +} + +/// Creates and imports the dfi file into the project from stream +/// contentStream - the stream which contains the source code in udl format.
+/// list - array of files to compile
+ClassMethod CreateDFI(contentStream As %CharacterStream, url As %String, ByRef list As %String) As %Status +{ + Set st = $$$OK + Try { + Set st = ..GetDFIName(contentStream, .name) + Return:$$$ISERR(st) st + + set list(name) = "" + + Set tDoc = ##class(%DeepSee.UI.FolderItemDocument).%New(name) + Set st = tDoc.ImportFromXML(contentStream) + Return:$$$ISERR(st) st + + Set st = tDoc.Save() + if st { + w !, "Imported " _ name, ! + } + Return:$$$ISERR(st) st + } Catch e { + Set st = e.AsStatus() + } + Return st +} + +/// Creates and imports mac, int, inc files into the project from stream +/// contentStream - the stream which contains the source code in udl format.
+/// url - the url where the file is located in the web.
+/// list - array of files to compile
+ClassMethod CreateRoutine(contentStream As %GlobalCharacterStream, url As %String, ByRef list As %String) As %Status +{ + Set st = ..GetRoutineName(contentStream, .name, .type) + do contentStream.Rewind() + + return:$$$ISERR(st) st + + if type = 1 { + set name = name _ ".mac" + } + elseif type = 2 { + set name = name _ ".inc" + } + elseif type = 3 { + set name = name _ ".int" + } + + set list(name) = "" + + Set rtn = ##class(%Routine).%New(name) + While 'contentStream.AtEnd { + Set line = contentStream.ReadLine() + If $Find(line, "ROUTINE") { + Continue + } + Do rtn.WriteLine(line) + } + + Set st = rtn.Save() + Return:$$$ISERR(st) st + + if st { + w !, "Imported " _ name, ! + } + Return st +} + +/// Creates and imports mac, int, inc files into the project from stream +/// contentStream - the stream which contains the source code in udl format.
+/// url - the url where the file is located in the web.
+/// ext - extension of the file
+/// list - array of files to compile
+ClassMethod CreateWebFile(contentStream As %GlobalCharacterStream, binaryStream As %Stream.FileCharacterGzip, url As %String, ext As %String, ByRef list As %String) As %Status +{ + Set st = $$$OK + Try + { + Set tDefaultApp=$system.CSP.GetDefaultApp($namespace)_"/" + set tCSPRootPath = $system.CSP.GetFileName(tDefaultApp) + + Set tFileName = $Piece($Piece(url,"?",1),"/",*) + + Set tCSPSubPath = $Piece(url,tDefaultApp,2,*) //still has filename and ? parameters in it + Set tCSPSubPath = $Piece(tCSPSubPath, tFileName,1) //now it does't + + set tFileDirectory = tCSPRootPath_tCSPSubPath + Set tFullFileName = tFileDirectory_tFileName + + //On Windows, tFullFileName will contain \ and / but CreateDirectoryChain() and + //LinkToFile() already normalize the paths accordingly to the OS for us so + //we don't have to worry about it. + If '##class(%File).CreateDirectoryChain(tFileDirectory) + { + Set st = $System.Status.Error(5001,"Could nor create path chain '"_tFileDirectory_"'") + Quit + } + + Set filestream = ##class(%Stream.FileCharacter).%New() + set st = filestream.LinkToFile(tFullFileName) + Quit:$System.Status.IsError(st) + + If $IsObject(contentStream) && ..IsWebFile(ext) + { + Set st=filestream.CopyFrom(contentStream) + } + Else + { + Set st=filestream.CopyFrom(binaryStream) + } + Quit:$System.Status.IsError(st) + + set st = filestream.%Save() + Quit:$System.Status.IsError(st) + + Write !, "Imported " _ tFullFileName, ! + } + Catch (oException) + { + Set st = oException.AsStatus() + } + + Quit st +} + +} +