Searching all of GitHub for "ponylang" and "pony-language" tags.

niclash · Aug 12, 2021 · 20d9049 · 20d9049
1 parent 533cc0d
commit 20d9049
Show file tree

Hide file tree

Showing 29 changed files with 209 additions and 38 deletions.
diff --git a/src/main/java/io/bali/ponyhub/StatisticsUtil.java b/src/main/java/io/bali/ponyhub/StatisticsUtil.java
@@ -14,13 +14,16 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static java.nio.file.StandardOpenOption.APPEND;
 import static java.nio.file.StandardOpenOption.CREATE;
 
 public class StatisticsUtil
 {
     private static final ScheduledExecutorService executor;
+    private static final Pattern semVerPattern = Pattern.compile( "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" );
 
     private static long githubMaxLimit;
 
@@ -33,24 +36,84 @@ public class StatisticsUtil
 
     public static void reportGithubRepository( ProjectVersion version )
     {
+        boolean newer = true;
         for( ProjectVersion v : recentUpdates )
         {
             if( v.repository().host().identity().equals( version.repository().host().identity() )
                 && v.organization().equals( version.organization() )
                 && v.name().equals( version.name() )
             )
             {
-                recentUpdates.remove( v );
+                if( isNewerVersion( version.version(), v.version() ) )
+                {
+                    recentUpdates.remove( v );
+                }
+                else
+                {
+                    newer = false;
+                }
                 break;
             }
         }
-        int ch = version.version().charAt( 0 );
-        if( ( ch >= '0' && ch <= '9' ) || ( ch == 'v' && version.version().matches( "[a-zA-Z]*[0-9.]*.*" ) ) )
+        if( newer && semVerPattern.matcher( version.version() ).matches() )
         {
             recentUpdates.add( version );
         }
     }
 
+    private static boolean isNewerVersion( String thisVersion, String previousVersion )
+    {
+        try
+        {
+            Matcher thisMatcher = semVerPattern.matcher( thisVersion );
+            if( !thisMatcher.matches() )
+            {
+                return false;
+            }
+            int thisMajor = Integer.parseInt( thisMatcher.group( 1 ) );
+            int thisMinor = Integer.parseInt( thisMatcher.group( 2 ) );
+            int thisPatch = Integer.parseInt( thisMatcher.group( 3 ) );
+            Matcher previousMatcher = semVerPattern.matcher( previousVersion );
+            if( !previousMatcher.matches() )
+            {
+                // Should never happen...
+                return false;
+            }
+            int prevMajor = Integer.parseInt( previousMatcher.group( 1 ) );
+            int prevMinor = Integer.parseInt( previousMatcher.group( 2 ) );
+            int prevPatch = Integer.parseInt( previousMatcher.group( 3 ) );
+            if( prevMajor > thisMajor )
+            {
+                return false;
+            }
+            if( prevMajor < thisMajor )
+            {
+                return true;
+            }
+            if( prevMinor > thisMinor )
+            {
+                return false;
+            }
+            if( prevMinor < thisMinor )
+            {
+                return true;
+            }
+            if( prevPatch > thisPatch )
+            {
+                return false;
+            }
+            if( prevPatch < thisPatch )
+            {
+                return true;
+            }
+        }
+        catch( Exception e )
+        {
+            e.printStackTrace();
+        }
+        return true;
+    }
+
     public static void reportGithubAccess( String maxLimit, String remainingLimit, String resetLimit )
     {
         cleanup();

diff --git a/src/main/java/io/bali/ponyhub/elastic/Hit.java b/src/main/java/io/bali/ponyhub/elastic/Hit.java
@@ -1,9 +1,11 @@
 package io.bali.ponyhub.elastic;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import io.bali.ponyhub.repositories.ProjectVersion;
 import io.bali.ponyhub.repositories.github.GitHubProjectVersion;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Hit
 {
     private String index;

diff --git a/src/main/java/io/bali/ponyhub/elastic/Hits.java b/src/main/java/io/bali/ponyhub/elastic/Hits.java
@@ -1,9 +1,11 @@
 package io.bali.ponyhub.elastic;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import io.bali.ponyhub.repositories.ProjectVersion;
 import java.util.List;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Hits
 {
     private Total total;

diff --git a/src/main/java/io/bali/ponyhub/elastic/SearchResult.java b/src/main/java/io/bali/ponyhub/elastic/SearchResult.java
@@ -1,7 +1,9 @@
 package io.bali.ponyhub.elastic;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class SearchResult
 {
     private int took;

diff --git a/src/main/java/io/bali/ponyhub/elastic/Shards.java b/src/main/java/io/bali/ponyhub/elastic/Shards.java
@@ -1,5 +1,8 @@
 package io.bali.ponyhub.elastic;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Shards
 {
     private int total;

diff --git a/src/main/java/io/bali/ponyhub/elastic/Total.java b/src/main/java/io/bali/ponyhub/elastic/Total.java
@@ -1,5 +1,8 @@
 package io.bali.ponyhub.elastic;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Total
 {
     private int value;

diff --git a/src/main/java/io/bali/ponyhub/repositories/BundleJson.java b/src/main/java/io/bali/ponyhub/repositories/BundleJson.java
@@ -15,7 +15,7 @@ public static class Dependency
 
         public String tag;
 
-        @JsonProperty("local-path")
+        @JsonProperty( "local-path" )
         public String local;
     }
 }
diff --git a/src/main/java/io/bali/ponyhub/repositories/RepositoryScan.java b/src/main/java/io/bali/ponyhub/repositories/RepositoryScan.java
@@ -51,9 +51,9 @@ public void run()
                 if( repo != null )
                 {
                     elastic.store( repo );
+                    List<RepositoryVersion> versions = repo.host().loadVersions( repo );
+                    versions.forEach( v -> VersionFetch.fetchVersion( repo, v, elastic, force ) );
                 }
-                List<RepositoryVersion> versions = repo.host().loadVersions( repo );
-                versions.forEach( v -> VersionFetch.fetchVersion( repo, v, elastic, force ) );
             }
         }
         catch( Throwable e )

diff --git a/src/main/java/io/bali/ponyhub/repositories/github/GitHub.java b/src/main/java/io/bali/ponyhub/repositories/github/GitHub.java
@@ -3,13 +3,15 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.bali.ponyhub.Main;
 import io.bali.ponyhub.StatisticsUtil;
 import io.bali.ponyhub.repositories.BundleJson;
 import io.bali.ponyhub.repositories.CorralDescriptor;
 import io.bali.ponyhub.repositories.ProjectVersion;
 import io.bali.ponyhub.repositories.Repository;
 import io.bali.ponyhub.repositories.RepositoryHost;
 import io.bali.ponyhub.repositories.RepositoryIdentity;
+import io.bali.ponyhub.repositories.RepositoryScan;
 import io.bali.ponyhub.repositories.RepositoryVersion;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -20,6 +22,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.StringJoiner;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import org.restlet.Response;
 import org.restlet.data.ChallengeResponse;
@@ -29,7 +34,6 @@
 import org.restlet.data.MediaType;
 import org.restlet.representation.Representation;
 import org.restlet.resource.ClientResource;
-import org.restlet.resource.ResourceException;
 import org.restlet.util.Series;
 
 import static java.time.temporal.ChronoUnit.MILLIS;
@@ -39,12 +43,20 @@
 public class GitHub
     implements RepositoryHost
 {
-    public static final RepositoryHost INSTANCE = new GitHub();
+    public static final RepositoryHost INSTANCE;
+    private static final ScheduledExecutorService executor;
 
     private GitHubApi api;
+
     private String token;
     private final ObjectMapper mapper;
 
+    static
+    {
+        executor = Executors.newScheduledThreadPool( 1 );
+        INSTANCE = new GitHub();
+    }
+
     private GitHub()
     {
         mapper = new ObjectMapper();
@@ -53,6 +65,7 @@ private GitHub()
         {
             initCredentials();
             api = load( "https://api.github.com", GitHubApi.class, emptyMap(), emptyMap() );
+            executor.scheduleAtFixedRate( this::searchForPonyRepositories, 0, 1440, TimeUnit.MINUTES );
         }
         catch( Exception e )
         {
@@ -62,6 +75,30 @@ private GitHub()
         }
     }
 
+    private void searchForPonyRepositories()
+    {
+        searchForPonyRepositories( "ponylang" );
+        searchForPonyRepositories( "pony-language" );
+    }
+
+    private void searchForPonyRepositories( String language )
+    {
+        try
+        {
+            Map<String, String> params = new HashMap<>();
+            params.put( "query", language );
+            GitHubRepositoriesSearchResult result = load( api.repository_search_url, GitHubRepositoriesSearchResult.class, params, emptyMap() );
+            for( GitHubRepository repo : result.items )
+            {
+                RepositoryScan.scanRepository( repo.identity(), Main.elastic, false );
+            }
+        }
+        catch( IOException e )
+        {
+            e.printStackTrace();
+        }
+    }
+
     @Override
     public ProjectVersion newProjectVersion( Repository repository, RepositoryVersion version, CorralDescriptor descriptor, BundleJson bundleJson, String readMe )
     {
@@ -121,7 +158,7 @@ public <T> T load( String urlTemplate, Class<T> resultType, Map<String, String>
         }
         catch( IOException e )
         {
-            System.err.println("Can not access " + url );
+            System.err.println( "Can not access " + url );
             throw e;
         }
     }
@@ -162,8 +199,8 @@ public List<RepositoryVersion> loadVersions( Repository repo )
         String template = ( (GitHubRepository) repo ).tags_url;
         GitHubTag[] tags = load( template, GitHubTag[].class, emptyMap(), emptyMap() );
         List<RepositoryVersion> versionTags = Arrays.stream( tags )
-                                                    .filter( v -> v.getName().matches( "[0-9]+(\\.[0-9]+)*" ) )
-                                                    .collect( Collectors.toList() );
+            .filter( v -> v.getName().matches( "[0-9]+(\\.[0-9]+)*" ) )
+            .collect( Collectors.toList() );
         String defaultBranch = repo.getDefaultBranch();
         versionTags.add( new NominalRepositoryVersion( defaultBranch ) );
         return versionTags;
@@ -252,12 +289,22 @@ private String fillTemplate( String urlTemplate, Map<String, String> requiredArg
         {
             url = url.replace( "{/" + opt.getKey() + "}", "/" + opt.getValue() );
         }
+        for( Map.Entry<String, String> opt : optionalArgs.entrySet() )
+        {
+            url = url.replace( "{" + opt.getKey() + "}", opt.getValue() );
+        }
         while( url.contains( "{/" ) )
         {
             int start = url.indexOf( "{/" );
             int end = url.indexOf( "}", start );
             url = url.substring( 0, start ) + url.substring( end + 1 );
         }
+        while( url.contains( "{" ) )
+        {
+            int start = url.indexOf( "{" );
+            int end = url.indexOf( "}", start );
+            url = url.substring( 0, start ) + url.substring( end + 1 );
+        }
         return url;
     }
 

diff --git a/src/main/java/io/bali/ponyhub/repositories/github/GitHubApi.java b/src/main/java/io/bali/ponyhub/repositories/github/GitHubApi.java
@@ -2,7 +2,7 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 
-@JsonIgnoreProperties
+@JsonIgnoreProperties( ignoreUnknown = true )
 public class GitHubApi
 {
     public String current_user_url;

diff --git a/src/main/java/io/bali/ponyhub/repositories/github/GitHubBlob.java b/src/main/java/io/bali/ponyhub/repositories/github/GitHubBlob.java
@@ -2,7 +2,7 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 
-@JsonIgnoreProperties
+@JsonIgnoreProperties( ignoreUnknown = true )
 public class GitHubBlob
 {
     public String sha;

diff --git a/src/main/java/io/bali/ponyhub/repositories/github/GitHubCommit.java b/src/main/java/io/bali/ponyhub/repositories/github/GitHubCommit.java
@@ -2,10 +2,9 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 
-@JsonIgnoreProperties
+@JsonIgnoreProperties( ignoreUnknown = true )
 public class GitHubCommit
 {
     public String sha;
     public String url;
-
 }
diff --git a/src/main/java/io/bali/ponyhub/repositories/github/GitHubContentEntry.java b/src/main/java/io/bali/ponyhub/repositories/github/GitHubContentEntry.java
@@ -1,9 +1,8 @@
 package io.bali.ponyhub.repositories.github;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
 
-@JsonIgnoreProperties
+@JsonIgnoreProperties( ignoreUnknown = true )
 public class GitHubContentEntry
 {
     public String name;

diff --git a/src/main/java/io/bali/ponyhub/repositories/github/GitHubOrganization.java b/src/main/java/io/bali/ponyhub/repositories/github/GitHubOrganization.java
@@ -3,7 +3,7 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import java.time.ZonedDateTime;
 
-@JsonIgnoreProperties
+@JsonIgnoreProperties( ignoreUnknown = true )
 public class GitHubOrganization
 {
     public String login;

diff --git a/src/main/java/io/bali/ponyhub/repositories/github/GitHubOwner.java b/src/main/java/io/bali/ponyhub/repositories/github/GitHubOwner.java
@@ -3,7 +3,7 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import io.bali.ponyhub.repositories.Owner;
 
-@JsonIgnoreProperties
+@JsonIgnoreProperties( ignoreUnknown = true )
 public class GitHubOwner
     implements Owner
 {

diff --git a/src/main/java/io/bali/ponyhub/repositories/github/GitHubProjectVersion.java b/src/main/java/io/bali/ponyhub/repositories/github/GitHubProjectVersion.java
@@ -9,7 +9,7 @@
 
 import static io.bali.ponyhub.repositories.IdentityResolver.escape;
 
-@JsonIgnoreProperties
+@JsonIgnoreProperties( ignoreUnknown = true )
 public class GitHubProjectVersion
     implements ProjectVersion
 {
@@ -29,8 +29,7 @@ public GitHubProjectVersion( Repository repository, RepositoryVersion version, C
         if( version.getName().equals( repository.getDefaultBranch() ) )
         {
             this.version = null;
-        }
-        else
+        } else
         {
             this.version = (GitHubTag) version;
         }