Doc update

.golangci.yaml

@@ -1,5 +1,3 @@
-run:
-  go: '1.23'
 linters:
   enable:
     - bodyclose

Makefile

@@ -6,6 +6,6 @@ update:
 generate:
 	go list -f '{{.Dir}}/...' -m | xargs go generate
 lint:
-	go list -f '{{.Dir}}/...' -m | xargs go run github.com/golangci/golangci-lint/cmd/golangci-lint@latest run -v --fix -c .golangci.yaml
+	go list -f '{{.Dir}}/...' -m | xargs go run github.com/golangci/golangci-lint/cmd/golangci-lint@latest run -j 4 --allow-parallel-runner -v --fix -c .golangci.yaml
 structalign:
 	go list -f '{{.Dir}}' -m | xargs -I {} sh -c 'cd {} && go run golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment -fix ./...'

README.md

@@ -3,19 +3,10 @@
 Backline is a minimalistic Internal Developer Portal (IDP) inspired by Backstage, built using Go and HTMX. It provides a flexible and extensible platform for exploring various developer resources.
 
 ## Table of Contents
+- [Demo](#demo)
 - [Features](#features)
 - [Why?](#why)
 - [Getting Started](#getting-started)
-- [Customization](#customization)
-- [Configuration](#configuration)
-  - [Basics](#basics)
-  - [Core Config](#core-config)
-  - [Repository Configuration](#repository-configuration)
-  - [Key-Value Storage Configuration](#key-value-storage-configuration)
-  - [Distributed Lock Configuration](#distributed-lock-configuration)
-  - [Job Scheduler Configuration](#job-scheduler-configuration)
-- [Existing Plugins](#existing-plugins)
-- [Plugin Development](#plugin-development)
 - [Contributing](#contributing)
 - [License](#license)
 
@@ -39,131 +30,102 @@ Some features may be disabled (for example "Scan" button)
 - **Lower learning curve**: Backline is designed to be easier to learn and use.
 - **Simpler upgrades**: Unlike Backstage, which requires scaffolding and is hard to upgrade, Backline isn't scaffolded and should work just by updating go.mod.
 - **Easy plugin development**:
-  - Plugins register themselves and do not require any code modifications.
+  - Plugins register themselves and do not require any code modifications (for now you need to include them into your main file as dependency. Later should be possible to just drop them as *.so files into plugins folder).
   - Plugins implement statically typed interfaces, making them easy to implement and detect version incompatibilities.
 - **Addressing Backstage issues**:
+  - Backstage does not support search on openapi and asyncapi specs (closed as not planned [backstage/backstage#22802](https://github.com/backstage/backstage/issues/22802)).
   - Backstage does not respect some relations like `apiProvidedBy` out of the box [backstage/backstage#25387](https://github.com/backstage/backstage/issues/25387).
   - Plugins use different HTTP clients that may not respect proxy settings [help-im-behind-a-corporate-proxy](https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/help-im-behind-a-corporate-proxy.md).
 
 ## Getting Started
 
-To get started with Backline with default plugins, run:
+To get started with Backline you need to add as least few plugins or specify implementations for core functionality
 
-```bash
-go run github.com/iamgoroot/backline/cmd/backline --config {your-config-location}/config.yaml
-```
+Let's start with a simple example of running just catalog and scanner with minimalistic set of plugins
+We will use the following plugins/features:
 
-This will run Backline with default plugins. You can also run it within your own application as a library:
+* Discovery for entities on file system
+* PostgreSQL plugin for storage (both entity and as Key-Value) 
+* PostgreSQL plugin for distributed lock (using pg_try_advisory_xact_lock)
+* Default plugin for scheduler (this is only option for now)
+* Default theme (with dark and light modes depending on OS theme)
 
 ```golang
-application := app.App{
-    // run catalog and scanner
-    PluggableDeps: app.PluggableDeps{
-        EntityDiscoveries: []core.Discovery{ // Add Location Readers so Backline knows how to read entities from different sources
-            &fs.Discovery{}, &github.Discovery{},
-        },
-        EntityRepo:        &store.Repo{},      // set repository plugin configurable with config file. supports pg and sqlite
-        KeyValStore:       &store.KV{},        // set key-value storage plugin configurable with config file. supports pg and sqlite
-        JobScheduler:      &store.Scheduler{}, // set job scheduler plugin configurable with config file. supports pg and sqlite
-        DistributedLocker: &store.Locker{},    // set distributed lock plugin configurable with config file. supports pg and sqlite
-    },
-    Plugins: []core.Plugin{
-        catalog.Plugin{}, // add web interface for catalog
-        oauth2.Plugin{},  // add oauth2 plugin for catalog authentication
-        stock.Theme{},    // add stock theme for catalog
-        &techdocs.Plugin{}, // add techdocs documentation plugin
-        &scanner.Plugin{}, // add scanner plugin to scan/read entities
-        openapiexplorer.Plugin{}, // add ability to display openapi specs on entity pages
-        rawdefinition.Plugin{},   // add ability to display raw API definitions on entity pages
-    },
-}
-
-err := application.Run()
+	application := app.App{
+		// run catalog and scanner
+
+		PluggableDeps: app.PluggableDeps{
+			EntityDiscoveries: []core.Discovery{    // Add Location Readers so backline knows how to read entities from different sources
+				&fs.Discovery{},                      // github repo to search entities
+			},
+			EntityRepo:         &pg.Repo{},         // use postgres implementation explicitly.
+			KeyValStore:        &kv.PgKV{},         // use postgres KV store explicitly.
+			JobScheduler:       &store.Scheduler{}, // job scheduler plugin. Basic implementation that uses KV store and Locker for scheduling and synchronizing tasks
+			DistributedLocker:  &store.Locker{},    // distributed lock plugin
+			ScannerPlugin:      &scanner.Plugin{},  // add scanner plugin to scan/read entities. 
+		},
+		Plugins: []core.Plugin{
+			catalog.Plugin{},                       // add web interface for catalog
+			stock.Theme{},                          // add stock theme for catalog
+		},
+	}
+
+  err := application.Run()
+  if err != nil {
+    log.Fatal(err)
+  }
 ```
 
-Check out the example config at `backline/cmd/config.yaml`.
-
-## Customization
-
-You can customize the appearance and functionality of Backline by modifying the configuration file or creating your own set of plugins.
+Although configuration can be done by populating plugin fields directly in the code, we will use a configuration file
+For sake of simplicity we will disable some security features (https, csrf, cors are enabled by default)
 
-### Run separate scanner in a separate service
 
-To run entity scan in a separate process:
+```yaml
+core:
+  server:
+    https:
+      disabled: true
+    csrf:
+      disabled: true
+    cors:
+      disabled: true
+  logger:
+    level: env:LOG_LEVEL
+    format: json
+  repo:
+    pg:
+      dsn: env:PG_DSN
+  kv:
+    pg:
+      dsn: env:PG_DSN
+  lock:
+    pg:
+      dsn: env:PG_DSN
+  scanner:
+    enableScanEndpoint: true
+    enableScanButton: true
+locations:
+  fs:
+    - "./entities"
 
-```bash
-go run github.com/iamgoroot/backline/cmd/scan_service --config {your-config-location}/config.yaml
 ```
 
-Or run entity scan as a library:
+You can populate config file directly or use env variables for config values
 
-```golang
-application := app.App{
-    Plugins: []app.Plugin{
-        &scanner.Plugin{},
-    },
-}
-err := application.Run()
+```env
+PG_DSN=postgresql://postgres:postgres@localhost:5432/backline?sslmode=disable
+LOG_LEVEL=INFO
 ```
 
-### Run catalog UI separately
-
-To run catalog UI in a separate process, run:
+Run the application
 
 ```bash
-go run github.com/iamgoroot/backline/cmd/webapp --config {your-config-location}/config.yaml
-```
-
-Or run catalog UI as a library in an existing service:
-
-```golang
-application := app.App{
-    Plugins: []app.Plugin{
-        catalog.Plugin{},
-        stock.Theme{},
-    },
-}
-
-err := application.Run()
+go run main.go --config {your-config-location}/config.yaml
 ```
 
-This will start Backline with no plugins except the default UI theme and catalog plugin. No OAuth2, no scanner, no OpenAPI explorer, no discovery.
+Open [http://localhost:8080](http://localhost:8080) and you should see the catalog UI. Click on `Scan entities` button to start scanning entities in directory `./entities`
 
-## Configuration
-
-### Basics
-
-Backline uses a YAML config file to configure itself. The config file allows you to specify default values and environment variable overrides.
-
-```yaml
-key: value1
-key_from_env: env:ENV_VAR_NAME
-key_with_default: env:ENV_VAR_NAME|default_value
-```
-
-In the example above, `key` will be set to `value1`, `key_from_env` will be set to the value of the environment variable `ENV_VAR_NAME`, and `key_with_default` will be set to the value of the environment variable `ENV_VAR_NAME` or `default_value` if the environment variable is not set.
-
-### Core Config
-
-Here's an example of a minimalistic core config:
-
-```yaml
-core:
-  server: # configure server ports, host, origins etc
-    port: env:PORT|8080
-    host: localhost
-    origins:
-    - http://localhost:8080
-  logger: # configure logging level and format
-    level: env:LOG_LEVEL|DEBUG
-    format: json
-  repo: # configure repository implementation
-    sqlite:
-      url: ":memory:" # use in-memory SQLite database for the simplest setup
-  kv: # configure key-value implementation
-    sqlite:
-      url: ":memory:" 
-```
+See more examples in `./examples` directory
 
 ## Core dependencies and Plugins
 

app/go.mod

@@ -3,25 +3,26 @@ module github.com/iamgoroot/backline/app
 go 1.24.0
 
 require (
-	github.com/iamgoroot/backline/pkg v0.0.0-20250110021921-01b18c16a3a1
+	github.com/iamgoroot/backline/pkg v0.0.0-20250219230517-431712432a82
 	github.com/labstack/echo/v4 v4.13.3
-	github.com/samber/slog-echo v1.15.1
-	golang.org/x/sync v0.11.0
+	github.com/samber/slog-echo v1.16.1
+	golang.org/x/sync v0.12.0
 )
 
 require (
-	github.com/goccy/go-yaml v1.15.23 // indirect
+	github.com/a-h/templ v0.3.857 // indirect
+	github.com/goccy/go-yaml v1.17.1 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/samber/lo v1.49.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
-	go.opentelemetry.io/otel v1.34.0 // indirect
-	go.opentelemetry.io/otel/trace v1.34.0 // indirect
-	golang.org/x/crypto v0.33.0 // indirect
-	golang.org/x/net v0.35.0 // indirect
-	golang.org/x/sys v0.30.0 // indirect
-	golang.org/x/text v0.22.0 // indirect
-	golang.org/x/time v0.10.0 // indirect
+	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel/trace v1.35.0 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/net v0.38.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
 )

app/go.sum

@@ -1,11 +1,13 @@
+github.com/a-h/templ v0.3.857 h1:6EqcJuGZW4OL+2iZ3MD+NnIcG7nGkaQeF2Zq5kf9ZGg=
+github.com/a-h/templ v0.3.857/go.mod h1:qhrhAkRFubE7khxLZHsBFHfX+gWwVNKbzKeF9GlPV4M=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/goccy/go-yaml v1.15.23 h1:WS0GAX1uNPDLUvLkNU2vXq6oTnsmfVFocjQ/4qA48qo=
-github.com/goccy/go-yaml v1.15.23/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/iamgoroot/backline/pkg v0.0.0-20250110021921-01b18c16a3a1 h1:X5E9C3tECBb/XdMj2Z2aRdfF6c8QUHiZKIWrJkf2+Z8=
-github.com/iamgoroot/backline/pkg v0.0.0-20250110021921-01b18c16a3a1/go.mod h1:eSKWBV+BjDxE4TYjIMgbqj0XdMNUPz67LyCu477qMrE=
+github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY=
+github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/iamgoroot/backline/pkg v0.0.0-20250219230517-431712432a82 h1:HB8ulDR/IgCbr+VkRiBfWyLUABTf9MBlTAde9A9aGao=
+github.com/iamgoroot/backline/pkg v0.0.0-20250219230517-431712432a82/go.mod h1:XW9xiDOtvm0aUCOF7doHAPrwWqXT75LzKfO/78wILVo=
 github.com/labstack/echo/v4 v4.13.3 h1:pwhpCPrTl5qry5HRdM5FwdXnhXSLSY+WE+YQSeCaafY=
 github.com/labstack/echo/v4 v4.13.3/go.mod h1:o90YNEeQWjDozo584l7AwhJMHN0bOC4tAfg+Xox9q5g=
 github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
@@ -18,30 +20,30 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew=
 github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o=
-github.com/samber/slog-echo v1.15.1 h1:mzeQNPYPxmpehIRtgQJRgJMVvrRbZHp5D2maxSljTBw=
-github.com/samber/slog-echo v1.15.1/go.mod h1:K21nbusPmai/MYm8PFactmZoFctkMmkeaTdXXyvhY1c=
+github.com/samber/slog-echo v1.16.1 h1:5Q5IUROkFqKcu/qJM/13AP1d3gd1RS+Q/4EvKQU1fuo=
+github.com/samber/slog-echo v1.16.1/go.mod h1:f+B3WR06saRXcaGRZ/I/UPCECDPqTUqadRIf7TmyRhI=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
 github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
-go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
-go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
-go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
-go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
-golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
-golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
-golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
-golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
-golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
-golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
+go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
+go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
+go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
+golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
-golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
-golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
+golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

app/routes.go

@@ -53,7 +53,7 @@ func setupMiddlewares(router *echo.Echo, cfg *CoreCfg) {
 }
 
 func setupCSRFMiddleware(router *echo.Echo, cfg *CoreCfg) {
-	if !cfg.Server.CSRF.Disabled {
+	if cfg.Server.CSRF.Disabled {
 		return
 	}
 
@@ -78,14 +78,16 @@ func setupCSRFMiddleware(router *echo.Echo, cfg *CoreCfg) {
 }
 
 func setupCORSMiddleware(router *echo.Echo, cfg *CoreCfg) {
-	if !cfg.Server.CORS.Disabled {
-		router.Use(middleware.CORSWithConfig(
-			middleware.CORSConfig{
-				AllowCredentials: true,
-				AllowOrigins:     cfg.Server.CORS.Origins,
-			}),
-		)
+	if cfg.Server.CORS.Disabled {
+		return
 	}
+
+	router.Use(middleware.CORSWithConfig(
+		middleware.CORSConfig{
+			AllowCredentials: true,
+			AllowOrigins:     cfg.Server.CORS.Origins,
+		}),
+	)
 }
 
 func writeErrResponse(c echo.Context, status int, err error) {