Merge branch 'refs/heads/feature/hotnews-operator-and-crd' into featu…

…re/operator-and-crd
werniq · Sep 6, 2024 · 3c37518 · 3c37518
2 parents 3e2e7c1 + d639466
commit 3c37518
Show file tree

Hide file tree

Showing 61 changed files with 5,730 additions and 203 deletions.
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -1,18 +1,16 @@
 version: '3'
 
 env:
-  # this variables can be specified when running task like this
-  # task publish DOCKER_IMAGE_NAME=new-image-title DOCKER_IMAGE_TAG=1.0.1
   DOCKER_IMAGE_NAME: '{{ .DOCKER_IMAGE_NAME | default "go-gator" }}'
   DOCKER_USERNAME:   '{{ .DOCKER_USERNAME | default "qniw984" }}'
-  DOCKER_IMAGE_TAG:  '{{ .DOCKER_IMAGE_TAG | default "1.3.0" }}'
-  TESTDIR_NAME:      '{{ .TESTDIR_NAME | default "pkg_tests" }}'
+  DOCKER_IMAGE_TAG:  '{{ .DOCKER_IMAGE_TAG | default "latest" }}'
   DOCKERHUB_USERNAME: '{{ .DOCKER_LOGIN_USERNAME }}'
   DOCKERHUB_PASSWORD: '{{ .DOCKERHUB_PASSWORD }}'
+  TESTDIR_NAME:      '{{ .TESTDIR_NAME | default "pkg_tests" }}'
 
 tasks:
   fmt:
-    desc: Check if files in the project need formatting
+    desc: Format all code in all subdirectories
     cmds:
       - go fmt ./...
 
@@ -56,7 +54,7 @@ tasks:
     label: Run all tests in application
     desc: This task runs all available test files from all child folders
     cmd: |
-      go test ./... 
+      go test ./... -v
 
   docker-build:
     desc: Build the docker image using Dockerfile with go-gator as a name
@@ -70,7 +68,7 @@ tasks:
     label: Pushing image to dockerhub
     desc: Push docker image to dockerhub
     deps:
-      - docker-build
+     - docker-build
     cmds:
       - docker login -u {{ .DOCKERHUB_USERNAME }} -p {{ .DOCKERHUB_PASSWORD }}
       - docker push {{ .DOCKER_USERNAME }}/{{ .DOCKER_IMAGE_NAME }}:{{ .DOCKER_IMAGE_TAG }}
@@ -84,3 +82,20 @@ tasks:
     desc: Remove news aggregator from kubernetes Cluster
     cmds:
       - kubectl delete -f templates/
+
+  start:
+    cmds:
+      - minikube start
+      - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml
+      - echo "Waiting for cert manager to working..."
+      - sleep 10
+      - cd operator
+      - make generate
+      - make manifests
+      - make install
+      - make deploy
+      - kubectl apply -f ../templates/
+
+  installTask:
+    cmds:
+      - go install github.com/go-task/task/v3/cmd/task@latest
diff --git a/cmd/parsers/data/sources.json b/cmd/parsers/data/sources.json
@@ -1 +1,22 @@
-[{"name":"washingtontimes","format":"xml","endpoint":"https://www.washingtontimes.com/rss/headlines/news/world"},{"name":"abc","format":"xml","endpoint":"https://abcnews.go.com/abcnews/internationalheadlines"},{"name":"bbc","format":"xml","endpoint":"https://feeds.bbci.co.uk/news/rss.xml"},{"name":"usatoday","format":"html","endpoint":"https://usatoday.com"}]
+[
+  {
+    "name": "washingtontimes",
+    "format": "xml",
+    "endpoint": "https://www.washingtontimes.com/rss/headlines/news/world"
+  },
+  {
+    "name": "abc",
+    "format": "xml",
+    "endpoint": "https://abcnews.go.com/abcnews/internationalheadlines"
+  },
+  {
+    "name": "bbc",
+    "format": "xml",
+    "endpoint": "https://feeds.bbci.co.uk/news/rss.xml"
+  },
+  {
+    "name": "usatoday",
+    "format": "html",
+    "endpoint": "https://usatoday.com"
+  }
+]
diff --git a/cmd/parsers/manage_sources.go b/cmd/parsers/manage_sources.go
@@ -2,7 +2,6 @@ package parsers
 
 import (
 	"encoding/json"
-	"errors"
 	"gogator/cmd/types"
 	"io"
 	"os"

diff --git a/devbox.json b/devbox.json
@@ -8,7 +8,7 @@
     "[email protected]",
     "[email protected]",
     "[email protected]",
-    "helm@0.9.0"
+    "kubernetes-helm@3.15.3"
   ],
   "shell": {
     "init_hook": [

diff --git a/devbox.lock b/devbox.lock
@@ -217,34 +217,6 @@
         }
       }
     },
-    "[email protected]": {
-      "last_modified": "2024-07-07T07:43:47Z",
-      "resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#helm",
-      "source": "devbox-search",
-      "version": "0.9.0",
-      "systems": {
-        "aarch64-linux": {
-          "outputs": [
-            {
-              "name": "out",
-              "path": "/nix/store/kc5gihv77ww1gbfvcgqzihjgswx9dx7m-helm-0.9.0",
-              "default": true
-            }
-          ],
-          "store_path": "/nix/store/kc5gihv77ww1gbfvcgqzihjgswx9dx7m-helm-0.9.0"
-        },
-        "x86_64-linux": {
-          "outputs": [
-            {
-              "name": "out",
-              "path": "/nix/store/faywfsglfwrlqrlmwsb69j658mpc7nwg-helm-0.9.0",
-              "default": true
-            }
-          ],
-          "store_path": "/nix/store/faywfsglfwrlqrlmwsb69j658mpc7nwg-helm-0.9.0"
-        }
-      }
-    },
     "[email protected]": {
       "last_modified": "2024-07-20T09:11:00Z",
       "resolved": "github:NixOS/nixpkgs/6e14bbce7bea6c4efd7adfa88a40dac750d80100#kubebuilder",
@@ -377,6 +349,54 @@
         }
       }
     },
+    "[email protected]": {
+      "last_modified": "2024-07-20T09:11:00Z",
+      "resolved": "github:NixOS/nixpkgs/6e14bbce7bea6c4efd7adfa88a40dac750d80100#kubernetes-helm",
+      "source": "devbox-search",
+      "version": "3.15.3",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/z4w7bnylg9h3f543yrf9bcwkxzfs82z2-kubernetes-helm-3.15.3",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/z4w7bnylg9h3f543yrf9bcwkxzfs82z2-kubernetes-helm-3.15.3"
+        },
+        "aarch64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/aa4jksq9ljgha8plw5cqyxf60n931dir-kubernetes-helm-3.15.3",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/aa4jksq9ljgha8plw5cqyxf60n931dir-kubernetes-helm-3.15.3"
+        },
+        "x86_64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/fbyg2gsaf4mwml30jrsd71c90yr6f5rz-kubernetes-helm-3.15.3",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/fbyg2gsaf4mwml30jrsd71c90yr6f5rz-kubernetes-helm-3.15.3"
+        },
+        "x86_64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/n4p0zh1s8jz9mqf1r1pki23kviq4waa7-kubernetes-helm-3.15.3",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/n4p0zh1s8jz9mqf1r1pki23kviq4waa7-kubernetes-helm-3.15.3"
+        }
+      }
+    },
     "[email protected]": {
       "last_modified": "2024-07-21T11:05:48Z",
       "resolved": "github:NixOS/nixpkgs/c19d62ad2265b16e2199c5feb4650fe459ca1c46#kustomize",

diff --git a/operator/Makefile b/operator/Makefile
@@ -1,6 +1,5 @@
 # Image URL to use all building/pushing image targets
-IMG ?= qniw984/feed-controller:1.0.0
-# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
+IMG ?= qniw984/hotnews-controller:1.3.8
 ENVTEST_K8S_VERSION = 1.30.0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)

diff --git a/operator/PROJECT b/operator/PROJECT
@@ -14,6 +14,7 @@ resources:
   controller: true
   domain: teamdev.com
   group: newsaggregator
+  kind: HotNews
   kind: Feed
   path: teamdev.com/go-gator/api/v1
   version: v1

diff --git a/operator/README.md b/operator/README.md
@@ -1,17 +1,31 @@
-# Operator
-This operator handles the creation, updating, and deletion of feeds on a news-aggregator server. 
-When a news feed is registered, a request is sent to the server to create a feed 
+### Operator Overview
+The `FeedReconciler` handles the creation, updating, and deletion of feeds on a news-aggregator server.
+When a news feed is registered, a request is sent to the server to create a feed
 with the specified name and endpoint.
 Similarly, when a news feed is updated or deleted, requests are made to update
 or remove the feed from the server accordingly.
 
-## Description
-This operator performs the management of feeds on a news-aggregator server by creating, 
-updating, or deleting feeds. 
-When the status of a Feed CRD changes (whether it is created, updated, or deleted), 
+The `HotNewsReconciler` is a Kubernetes controller responsible for managing the `HotNews` custom resource. 
+It ensures that the status of the `HotNews` resource is always up-to-date by interacting with a news aggregator server
+to fetch the latest news based on specified parameters.
+
+### Description
+
+The `FeedReconciler` performs the management of feeds on a news-aggregator server by creating,
+updating, or deleting feeds.
+When the status of a Feed CRD changes (whether it is created, updated, or deleted),
 the operator sends a request to the news-aggregator server to perform the corresponding action
 with the specified name and endpoint.
 
+The `HotNewsReconciler` performs the following key tasks:
+- Monitors the status of `HotNews` custom resources and triggers updates whenever the resource is created, updated, or deleted.
+- Sends requests to a news aggregator server to retrieve news articles based on specified keywords, date ranges, and sources.
+- Validates the input parameters (keywords, date range, and feeds) before making a request to ensure accuracy.
+- Watches for changes in the `ConfigMap` containing feed groups and in the `Feed` CRD, and updates the `HotNews` resource accordingly.
+- Handles the creation, update, and deletion of `HotNews` resources, ensuring that the Kubernetes cluster state aligns with the desired state.
+
+The reconciler requeues and reconciles every 24 hours to keep the news content fresh and updated.
+
 ## Getting Started
 
 ### Prerequisites

diff --git a/operator/api/v1/doc.go b/operator/api/v1/doc.go
@@ -1,15 +1,29 @@
 /*
-Package v1 contains CRD definitions for the Feed resource, and Validation webhook that ensures uniqueness of the
+Package v1 contains CRD definitions for the Feed and HotNews resources. Validation and mutating webhooks that ensures uniqueness of the
 feed.
 
 This package includes the following components:
 - Custom Resource Definition for the Feed resource.
 - Webhook for validating Feed resources.
+- CRD for the HotNews resource.
+- Validating and mutating webhooks for the HotNews resource.
 
 The Feed resource represents a news feed with a name and a link.
 
 The webhook ensures that the Feed resources meet the required validation criteria, in particular:
 - Name should be unique within namespace, and should not be more than 20 symbols
 - Endpoint of feed should be either http or https url.
+
+The HotNews resource represents a group of news feeds with a name and a list of feed names.
+This CRD allows us to create hot news based on available Feeds.
+
+The mutating webhook sets the default values for the HotNews resource, if they are not specified, like:
+- Spec.SummaryConfig.TitlesCount should be 10 by default.
+
+The validating webhook ensures that the HotNews resources meet the required validation criteria, in particular:
+- Either Feeds or FeedGroups should be specified.
+- DateStart should be before DateEnd.
+- All feed names should be correct.
+- Keywords are not empty.
 */
 package v1
diff --git a/operator/api/v1/feed_types.go b/operator/api/v1/feed_types.go
@@ -117,10 +117,6 @@ type FeedList struct {
 	Items           []Feed `json:"items"`
 }
 
-func init() {
-	SchemeBuilder.Register(&Feed{}, &FeedList{})
-}
-
 // SetCreatedCondition sets the created condition of the feed to Created
 func (r *Feed) SetCreatedCondition(reason string) {
 	r.setCondition(TypeFeedCreated, createdReason, reason, FeedCreated)