From 6232fa0cbdf6de6aab82b3ae267a1caeb636e17f Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Sun, 15 Feb 2026 08:27:42 +0530 Subject: [PATCH 1/5] [FEAT]: Add Gin full-text search implementation with Typesense integration --- typesense-gin-full-text-search/.env.example | 9 ++ typesense-gin-full-text-search/README.md | 120 +++++++++++++++++ typesense-gin-full-text-search/go.mod | 51 ++++++++ typesense-gin-full-text-search/go.sum | 122 ++++++++++++++++++ .../routes/search.go | 55 ++++++++ typesense-gin-full-text-search/server.go | 34 +++++ typesense-gin-full-text-search/utils/env.go | 44 +++++++ .../utils/typesense.go | 22 ++++ 8 files changed, 457 insertions(+) create mode 100644 typesense-gin-full-text-search/.env.example create mode 100644 typesense-gin-full-text-search/README.md create mode 100644 typesense-gin-full-text-search/go.mod create mode 100644 typesense-gin-full-text-search/go.sum create mode 100644 typesense-gin-full-text-search/routes/search.go create mode 100644 typesense-gin-full-text-search/server.go create mode 100644 typesense-gin-full-text-search/utils/env.go create mode 100644 typesense-gin-full-text-search/utils/typesense.go diff --git a/typesense-gin-full-text-search/.env.example b/typesense-gin-full-text-search/.env.example new file mode 100644 index 0000000..d8b0936 --- /dev/null +++ b/typesense-gin-full-text-search/.env.example @@ -0,0 +1,9 @@ +# Server Configuration +PORT=3000 + +# Typesense Configuration +TYPESENSE_HOST=localhost +TYPESENSE_PORT=8108 +TYPESENSE_PROTOCOL=http +TYPESENSE_API_KEY=xyz +TYPESENSE_COLLECTION=books diff --git a/typesense-gin-full-text-search/README.md b/typesense-gin-full-text-search/README.md new file mode 100644 index 0000000..c256d50 --- /dev/null +++ b/typesense-gin-full-text-search/README.md @@ -0,0 +1,120 @@ +# Gin Full-Text Search with Typesense + +A RESTful search API built with Go Gin framework and Typesense, featuring full-text search capabilities with environment-based configuration. + +## Tech Stack + +- Go 1.19+ +- Gin Web Framework +- Typesense + +## Prerequisites + +- Go 1.19+ installed +- Docker (for running Typesense locally). Alternatively, you can use a Typesense Cloud cluster. +- Basic knowledge of Go and REST APIs. + +## Quick Start + +### 1. Clone the repository + +```bash +git clone https://github.com/typesense/code-samples.git +cd typesense-gin-full-text-search +``` + +### 2. Install dependencies + +```bash +go mod download +``` + +### 3. Set up environment variables + +Create a `.env` file in the project root with the following content: + +```env +# Server Configuration +PORT=3000 + +# Typesense Configuration +TYPESENSE_HOST=localhost +TYPESENSE_PORT=8108 +TYPESENSE_PROTOCOL=http +TYPESENSE_API_KEY=xyz +TYPESENSE_COLLECTION=books +``` + +### 4. Project Structure + +```text +├── routes +│ └── search.go +├── utils +│ ├── env.go +│ └── typesense.go +├── server.go +├── go.mod +└── .env +``` + +### 5. Start the development server + +**Standard mode:** + +```bash +go run server.go +``` + +**Hot reload mode (recommended for development):** + +First, install CompileDaemon: + +```bash +go install github.com/githubnemo/CompileDaemon@latest +``` + +Then run: + +```bash +CompileDaemon --build="go build -o server ." --command="./server" +``` + +The server will automatically restart when you make changes to any Go file. + +Open [http://localhost:3000](http://localhost:3000) in your browser. + +### 6. Search API Endpoint + +**Search:** + +```bash +GET /search?q= +``` + +Example: + +```bash +curl "http://localhost:3000/search?q=harry" +``` + +### 7. Deployment + +Set env variables to point the app to the Typesense Cluster: + +```env +# Server Configuration +PORT=3000 + +# Typesense Configuration +TYPESENSE_HOST=xxx.typesense.net +TYPESENSE_PORT=443 +TYPESENSE_PROTOCOL=https +TYPESENSE_API_KEY=your-production-api-key +TYPESENSE_COLLECTION=books +``` + +- Configure CORS middleware for specific origins. +- Configure gin to run in release mode. +- Add some sort of authentication to the API. +- Add rate limiting to the API. diff --git a/typesense-gin-full-text-search/go.mod b/typesense-gin-full-text-search/go.mod new file mode 100644 index 0000000..d1c8348 --- /dev/null +++ b/typesense-gin-full-text-search/go.mod @@ -0,0 +1,51 @@ +module github.com/typesense/code-samples/typesense-gin-full-text-search + +go 1.25.0 + +require ( + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/fatih/color v1.9.0 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect + github.com/gin-contrib/cors v1.7.6 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/gin-gonic/gin v1.11.0 // indirect + github.com/githubnemo/CompileDaemon v1.4.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/radovskyb/watcher v1.0.7 // indirect + github.com/sony/gobreaker v1.0.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/typesense/typesense-go/v4 v4.0.0-alpha2 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.uber.org/mock v0.6.0 // indirect + golang.org/x/arch v0.24.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/mod v0.33.0 // indirect + golang.org/x/net v0.50.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + golang.org/x/tools v0.42.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/typesense-gin-full-text-search/go.sum b/typesense-gin-full-text-search/go.sum new file mode 100644 index 0000000..07e438c --- /dev/null +++ b/typesense-gin-full-text-search/go.sum @@ -0,0 +1,122 @@ +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/cors v1.7.6 h1:3gQ8GMzs1Ylpf70y8bMw4fVpycXIeX1ZemuSQIsnQQY= +github.com/gin-contrib/cors v1.7.6/go.mod h1:Ulcl+xN4jel9t1Ry8vqph23a60FwH9xVLd+3ykmTjOk= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= +github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= +github.com/githubnemo/CompileDaemon v1.4.0 h1:z96Qu4tj+RzRfF+L7f1O6E8ion5JQlisWeXWc2wzwDQ= +github.com/githubnemo/CompileDaemon v1.4.0/go.mod h1:/G125r3YBIp6rcXtCZfiEHwFzcl7GSsNSwylxSNrkMA= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE= +github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg= +github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ= +github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/typesense/typesense-go/v4 v4.0.0-alpha2 h1:L9icvEu+N9ARlmwh0eM3Cfc/zZLTNwn5nXS1Z4bztEk= +github.com/typesense/typesense-go/v4 v4.0.0-alpha2/go.mod h1:Y880M+mG3T9jthku5MJBmfXrrc2wyE6ZotLOOADZx9Q= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= +golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/typesense-gin-full-text-search/routes/search.go b/typesense-gin-full-text-search/routes/search.go new file mode 100644 index 0000000..ec9882a --- /dev/null +++ b/typesense-gin-full-text-search/routes/search.go @@ -0,0 +1,55 @@ +package routes + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/typesense/code-samples/typesense-gin-full-text-search/utils" + "github.com/typesense/typesense-go/v4/typesense/api" + "github.com/typesense/typesense-go/v4/typesense/api/pointer" +) + +// SetupSearchRoutes configures all search-related routes +func SetupSearchRoutes(router *gin.Engine) { + // Simple search endpoint + router.GET("/search", searchBooks) +} + +// searchBooks handles the search request +func searchBooks(c *gin.Context) { + query := c.Query("q") + if query == "" { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "Search query parameter 'q' is required", + }) + return + } + + // Create search parameters + searchParams := &api.SearchCollectionParams{ + Q: pointer.String(query), + QueryBy: pointer.String("title,authors"), + } + + // Perform search using the Typesense client + result, err := utils.Client.Collection(utils.BookCollection).Documents().Search(c.Request.Context(), searchParams) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "Search failed: " + err.Error(), + "debug": gin.H{ + "collection": utils.BookCollection, + "query": query, + "server_url": utils.GetServerURL(), + }, + }) + return + } + + // Return search results + c.JSON(http.StatusOK, gin.H{ + "query": query, + "results": *result.Hits, + "found": *result.Found, + "took": result.SearchTimeMs, + }) +} diff --git a/typesense-gin-full-text-search/server.go b/typesense-gin-full-text-search/server.go new file mode 100644 index 0000000..94c1f99 --- /dev/null +++ b/typesense-gin-full-text-search/server.go @@ -0,0 +1,34 @@ +package main + +import ( + "github.com/gin-contrib/cors" + "github.com/gin-gonic/gin" + "github.com/typesense/code-samples/typesense-gin-full-text-search/routes" + "github.com/typesense/code-samples/typesense-gin-full-text-search/utils" +) + +func main() { + router := gin.Default() + + // CORS middleware + router.Use(cors.New(cors.Config{ + AllowOrigins: []string{"*"}, + AllowMethods: []string{"GET", "OPTIONS"}, + AllowHeaders: []string{"Origin", "Content-Type", "Accept", "Authorization"}, + ExposeHeaders: []string{"Content-Length"}, + AllowCredentials: true, + })) + + // Health check endpoint + router.GET("/ping", func(c *gin.Context) { + c.JSON(200, gin.H{ + "message": "pong", + }) + }) + + // Setup search routes + routes.SetupSearchRoutes(router) + + port := utils.GetEnv("PORT", "3000") + router.Run(":" + port) +} diff --git a/typesense-gin-full-text-search/utils/env.go b/typesense-gin-full-text-search/utils/env.go new file mode 100644 index 0000000..0fa36d5 --- /dev/null +++ b/typesense-gin-full-text-search/utils/env.go @@ -0,0 +1,44 @@ +package utils + +import ( + "log" + "os" + "strconv" + + "github.com/joho/godotenv" +) + +// init() function runs automatically when the package is imported +func init() { + // Load .env file + if err := godotenv.Load(); err != nil { + log.Println("No .env file found, using environment variables") + } +} + +// Helper functions to read environment variables with defaults +func GetEnv(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +func GetEnvAsInt(key string, defaultValue int) int { + if value := os.Getenv(key); value != "" { + if intValue, err := strconv.Atoi(value); err == nil { + return intValue + } + } + return defaultValue +} + +func GetServerURL() string { + protocol := GetEnv("TYPESENSE_PROTOCOL", "http") + host := GetEnv("TYPESENSE_HOST", "localhost") + port := GetEnvAsInt("TYPESENSE_PORT", 8108) + return protocol + "://" + host + ":" + strconv.Itoa(port) +} + +// Collection name for books +var BookCollection = GetEnv("TYPESENSE_COLLECTION", "books") diff --git a/typesense-gin-full-text-search/utils/typesense.go b/typesense-gin-full-text-search/utils/typesense.go new file mode 100644 index 0000000..2650eee --- /dev/null +++ b/typesense-gin-full-text-search/utils/typesense.go @@ -0,0 +1,22 @@ +package utils + +import ( + "log" + + "github.com/typesense/typesense-go/v4/typesense" +) + +var Client *typesense.Client + +func init() { + apiKey := GetEnv("TYPESENSE_API_KEY", "xyz") + serverURL := GetServerURL() + + // Create client with simple configuration + Client = typesense.NewClient( + typesense.WithServer(serverURL), + typesense.WithAPIKey(apiKey), + ) + + log.Printf("Typesense Client created successfully") +} From b8e1cac443cb644e08bc959a177d6325be8fbf7a Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Tue, 17 Feb 2026 22:46:06 +0530 Subject: [PATCH 2/5] [IMPROVEMENT]: Add collection initialization, bulk data import, and search enhancements to Gin implementation --- .gitignore | 5 +- .../routes/search.go | 15 +- typesense-gin-full-text-search/server.go | 20 +++ .../utils/collections.go | 46 ++++++ .../utils/data_import.go | 132 ++++++++++++++++++ 5 files changed, 211 insertions(+), 7 deletions(-) create mode 100644 typesense-gin-full-text-search/utils/collections.go create mode 100644 typesense-gin-full-text-search/utils/data_import.go diff --git a/.gitignore b/.gitignore index a906e02..929d39f 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,7 @@ pnpm-lock.yaml .astro/ # react-native -.expo \ No newline at end of file +.expo + +# Data +books.jsonl \ No newline at end of file diff --git a/typesense-gin-full-text-search/routes/search.go b/typesense-gin-full-text-search/routes/search.go index ec9882a..dec7428 100644 --- a/typesense-gin-full-text-search/routes/search.go +++ b/typesense-gin-full-text-search/routes/search.go @@ -27,8 +27,10 @@ func searchBooks(c *gin.Context) { // Create search parameters searchParams := &api.SearchCollectionParams{ - Q: pointer.String(query), - QueryBy: pointer.String("title,authors"), + Q: pointer.String(query), + QueryBy: pointer.String("title,authors"), + QueryByWeights: pointer.String("2,1"), // Title matches are weighted 2x more than author matches + FacetBy: pointer.String("authors,publication_year,average_rating"), // Get facet counts for filtering } // Perform search using the Typesense client @@ -47,9 +49,10 @@ func searchBooks(c *gin.Context) { // Return search results c.JSON(http.StatusOK, gin.H{ - "query": query, - "results": *result.Hits, - "found": *result.Found, - "took": result.SearchTimeMs, + "query": query, + "results": *result.Hits, + "found": *result.Found, + "took": result.SearchTimeMs, + "facet_counts": result.FacetCounts, }) } diff --git a/typesense-gin-full-text-search/server.go b/typesense-gin-full-text-search/server.go index 94c1f99..b85c828 100644 --- a/typesense-gin-full-text-search/server.go +++ b/typesense-gin-full-text-search/server.go @@ -1,6 +1,10 @@ package main import ( + "context" + "log" + "time" + "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" "github.com/typesense/code-samples/typesense-gin-full-text-search/routes" @@ -8,6 +12,22 @@ import ( ) func main() { + // Initialize collections before starting the server + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := utils.InitializeCollections(ctx); err != nil { + log.Fatalf("Failed to initialize collections: %v", err) + } + + // Initialize data if collection is empty + // This is idempotent - only imports if collection has no documents + dataFile := "books.jsonl" + if err := utils.InitializeDataIfEmpty(ctx, utils.BookCollection, dataFile); err != nil { + log.Printf("Warning: Failed to initialize data: %v", err) + log.Println("Server will continue, but collection may be empty") + } + router := gin.Default() // CORS middleware diff --git a/typesense-gin-full-text-search/utils/collections.go b/typesense-gin-full-text-search/utils/collections.go new file mode 100644 index 0000000..4186ce8 --- /dev/null +++ b/typesense-gin-full-text-search/utils/collections.go @@ -0,0 +1,46 @@ +package utils + +import ( + "context" + "log" + + "github.com/typesense/typesense-go/v4/typesense/api" + "github.com/typesense/typesense-go/v4/typesense/api/pointer" +) + +// InitializeCollections ensures all required collections exist +// This is idempotent - safe to call multiple times +func InitializeCollections(ctx context.Context) error { + log.Println("Initializing Typesense collections...") + + // Define the books collection schema + booksSchema := &api.CollectionSchema{ + Name: BookCollection, + Fields: []api.Field{ + {Name: "title", Type: "string", Facet: pointer.False()}, + {Name: "authors", Type: "string[]", Facet: pointer.True()}, + {Name: "publication_year", Type: "int32", Facet: pointer.True()}, + {Name: "average_rating", Type: "float", Facet: pointer.True()}, + {Name: "image_url", Type: "string", Facet: pointer.False()}, + {Name: "ratings_count", Type: "int32", Facet: pointer.True()}, + }, + DefaultSortingField: pointer.String("ratings_count"), + } + + // Try to retrieve the collection to check if it exists + _, err := Client.Collection(BookCollection).Retrieve(ctx) + if err != nil { + // Collection doesn't exist, create it + log.Printf("Collection '%s' not found, creating...", BookCollection) + _, err = Client.Collections().Create(ctx, booksSchema) + if err != nil { + log.Printf("Failed to create collection '%s': %v", BookCollection, err) + return err + } + log.Printf("Collection '%s' created successfully", BookCollection) + } else { + log.Printf("Collection '%s' already exists, skipping creation", BookCollection) + } + + return nil +} diff --git a/typesense-gin-full-text-search/utils/data_import.go b/typesense-gin-full-text-search/utils/data_import.go new file mode 100644 index 0000000..c71c369 --- /dev/null +++ b/typesense-gin-full-text-search/utils/data_import.go @@ -0,0 +1,132 @@ +package utils + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "log" + "os" + + "github.com/typesense/typesense-go/v4/typesense/api" + "github.com/typesense/typesense-go/v4/typesense/api/pointer" +) + +// ImportDocumentsFromJSONL imports documents from a JSONL file in bulk +// This is the production-ready way to load initial data +func ImportDocumentsFromJSONL(ctx context.Context, collectionName, filePath string) error { + log.Printf("Starting bulk import from %s to collection '%s'...", filePath, collectionName) + + // Read the JSONL file + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + + // Parse each line as a JSON document + scanner := bufio.NewScanner(file) + var documents []interface{} + lineCount := 0 + + for scanner.Scan() { + var doc map[string]interface{} + if err := json.Unmarshal(scanner.Bytes(), &doc); err != nil { + log.Printf("Warning: skipping invalid JSON line: %v", err) + continue + } + documents = append(documents, doc) + lineCount++ + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("error reading file: %w", err) + } + + log.Printf("Read %d documents from file", lineCount) + + // Import documents in bulk using the import API + // BatchSize controls how many documents are processed at once + importParams := &api.ImportDocumentsParams{ + BatchSize: pointer.Int(100), // Process in batches of 100 + } + + // The Import method accepts []interface{} containing document maps + results, err := Client.Collection(collectionName).Documents().Import( + ctx, + documents, + importParams, + ) + + if err != nil { + return fmt.Errorf("bulk import failed: %w", err) + } + + // Count successes and failures + successCount := 0 + failureCount := 0 + + for _, result := range results { + if result.Success { + successCount++ + } else { + failureCount++ + // Log first few errors for debugging + if failureCount <= 5 { + log.Printf("Import error: %s", result.Error) + } + } + } + + log.Printf("Bulk import completed: %d succeeded, %d failed", successCount, failureCount) + + if failureCount > 0 && failureCount > lineCount/2 { + // Only error if more than half failed + return fmt.Errorf("bulk import had too many failures: %d out of %d", failureCount, lineCount) + } + + return nil +} + +// CheckCollectionDocumentCount returns the number of documents in a collection +func CheckCollectionDocumentCount(ctx context.Context, collectionName string) (int, error) { + collection, err := Client.Collection(collectionName).Retrieve(ctx) + if err != nil { + return 0, fmt.Errorf("failed to retrieve collection: %w", err) + } + + return int(*collection.NumDocuments), nil +} + +// InitializeDataIfEmpty checks if collection is empty and imports data if needed +// This is idempotent - safe to run on every startup +func InitializeDataIfEmpty(ctx context.Context, collectionName, dataFilePath string) error { + log.Printf("Checking if collection '%s' needs data initialization...", collectionName) + + // Check current document count + count, err := CheckCollectionDocumentCount(ctx, collectionName) + if err != nil { + return fmt.Errorf("failed to check document count: %w", err) + } + + if count > 0 { + log.Printf("Collection '%s' already has %d documents, skipping import", collectionName, count) + return nil + } + + log.Printf("Collection '%s' is empty, importing data from %s", collectionName, dataFilePath) + + // Import data + if err := ImportDocumentsFromJSONL(ctx, collectionName, dataFilePath); err != nil { + return fmt.Errorf("failed to import data: %w", err) + } + + // Verify import + newCount, err := CheckCollectionDocumentCount(ctx, collectionName) + if err != nil { + return fmt.Errorf("failed to verify import: %w", err) + } + + log.Printf("Data import successful: collection '%s' now has %d documents", collectionName, newCount) + return nil +} From b7650657d3091b42db0b5d523d567e25908dc26a Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Wed, 25 Feb 2026 08:04:30 +0530 Subject: [PATCH 3/5] Second revision: Add PostgreSQL database integration with CRUD operations and automatic Typesense synchronization --- typesense-gin-full-text-search/.env.example | 7 + typesense-gin-full-text-search/go.mod | 8 + typesense-gin-full-text-search/go.sum | 17 + typesense-gin-full-text-search/models/book.go | 39 +++ .../routes/books.go | 196 +++++++++++ .../routes/search.go | 68 ++++ typesense-gin-full-text-search/server.go | 29 +- .../utils/database.go | 140 ++++++++ typesense-gin-full-text-search/utils/sync.go | 326 ++++++++++++++++++ .../utils/sync_worker.go | 133 +++++++ 10 files changed, 956 insertions(+), 7 deletions(-) create mode 100644 typesense-gin-full-text-search/models/book.go create mode 100644 typesense-gin-full-text-search/routes/books.go create mode 100644 typesense-gin-full-text-search/utils/database.go create mode 100644 typesense-gin-full-text-search/utils/sync.go create mode 100644 typesense-gin-full-text-search/utils/sync_worker.go diff --git a/typesense-gin-full-text-search/.env.example b/typesense-gin-full-text-search/.env.example index d8b0936..0afb003 100644 --- a/typesense-gin-full-text-search/.env.example +++ b/typesense-gin-full-text-search/.env.example @@ -7,3 +7,10 @@ TYPESENSE_PORT=8108 TYPESENSE_PROTOCOL=http TYPESENSE_API_KEY=xyz TYPESENSE_COLLECTION=books + +# Database Credentials +DB_HOST=xxx +DB_PORT=xxx +DB_USER=xxx +DB_PASSWORD=xxx +DB_NAME=xxx \ No newline at end of file diff --git a/typesense-gin-full-text-search/go.mod b/typesense-gin-full-text-search/go.mod index d1c8348..0856fc3 100644 --- a/typesense-gin-full-text-search/go.mod +++ b/typesense-gin-full-text-search/go.mod @@ -21,6 +21,12 @@ require ( github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.8.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect @@ -48,4 +54,6 @@ require ( golang.org/x/text v0.34.0 // indirect golang.org/x/tools v0.42.0 // indirect google.golang.org/protobuf v1.36.11 // indirect + gorm.io/driver/postgres v1.6.0 // indirect + gorm.io/gorm v1.31.1 // indirect ) diff --git a/typesense-gin-full-text-search/go.sum b/typesense-gin-full-text-search/go.sum index 07e438c..b73f9bd 100644 --- a/typesense-gin-full-text-search/go.sum +++ b/typesense-gin-full-text-search/go.sum @@ -39,6 +39,18 @@ github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7Lk github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo= +github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -81,6 +93,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -120,3 +133,7 @@ google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= +gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= diff --git a/typesense-gin-full-text-search/models/book.go b/typesense-gin-full-text-search/models/book.go new file mode 100644 index 0000000..49295ae --- /dev/null +++ b/typesense-gin-full-text-search/models/book.go @@ -0,0 +1,39 @@ +package models + +import ( + "fmt" + "time" + + "gorm.io/gorm" +) + +type Book struct { + ID uint `gorm:"primaryKey" json:"id"` + Title string `json:"title"` + Authors []string `gorm:"serializer:json" json:"authors"` + PublicationYear int `json:"publication_year"` + AverageRating float64 `json:"average_rating"` + ImageUrl string `json:"image_url"` + RatingsCount int `json:"ratings_count"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt gorm.DeletedAt `gorm:"index" json:"deleted_at,omitempty"` +} + +func (b *Book) GetTypesenseID() string { + return fmt.Sprintf("book_%d", b.ID) +} + +func (b *Book) BeforeCreate(tx *gorm.DB) error { + return nil +} + +func (b *Book) BeforeUpdate(tx *gorm.DB) error { + b.UpdatedAt = time.Now() + return nil +} + +func (b *Book) BeforeDelete(tx *gorm.DB) error { + b.UpdatedAt = time.Now() + return nil +} diff --git a/typesense-gin-full-text-search/routes/books.go b/typesense-gin-full-text-search/routes/books.go new file mode 100644 index 0000000..b39e502 --- /dev/null +++ b/typesense-gin-full-text-search/routes/books.go @@ -0,0 +1,196 @@ +package routes + +import ( + "context" + "log" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/typesense/code-samples/typesense-gin-full-text-search/models" + "github.com/typesense/code-samples/typesense-gin-full-text-search/utils" +) + +// SetupBookRoutes configures all book CRUD routes +func SetupBookRoutes(router *gin.Engine) { + books := router.Group("/books") + { + books.POST("", createBook) + books.GET("/:id", getBook) + books.GET("", getAllBooks) + books.PUT("/:id", updateBook) + books.DELETE("/:id", deleteBook) + } +} + +// createBook creates a new book in the database and syncs to Typesense +func createBook(c *gin.Context) { + var book models.Book + + if err := c.ShouldBindJSON(&book); err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "Invalid request body: " + err.Error(), + }) + return + } + + // Save to database (source of truth) + if err := utils.SaveBook(c.Request.Context(), &book); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "Failed to create book: " + err.Error(), + }) + return + } + + // Sync to Typesense asynchronously (non-blocking) + go func(bookCopy models.Book) { + ctx := context.Background() + if err := utils.SyncBookOnUpdate(ctx, &bookCopy); err != nil { + log.Printf("Async Typesense sync failed for book %d: %v", bookCopy.ID, err) + } + }(book) + + c.JSON(http.StatusCreated, gin.H{ + "message": "Book created successfully", + "book": book, + }) +} + +// getBook retrieves a single book by ID +func getBook(c *gin.Context) { + var uri struct { + ID uint `uri:"id" binding:"required"` + } + + if err := c.ShouldBindUri(&uri); err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "Invalid book ID", + }) + return + } + + book, err := utils.GetBookByID(c.Request.Context(), uri.ID) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{ + "error": "Book not found", + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "book": book, + }) +} + +// getAllBooks retrieves all books from the database +func getAllBooks(c *gin.Context) { + books, err := utils.GetAllBooks(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "Failed to fetch books: " + err.Error(), + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "count": len(books), + "books": books, + }) +} + +// updateBook updates an existing book and syncs to Typesense +func updateBook(c *gin.Context) { + var uri struct { + ID uint `uri:"id" binding:"required"` + } + + if err := c.ShouldBindUri(&uri); err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "Invalid book ID", + }) + return + } + + // Fetch existing book + book, err := utils.GetBookByID(c.Request.Context(), uri.ID) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{ + "error": "Book not found", + }) + return + } + + // Bind updated data directly to existing book + if err := c.ShouldBindJSON(&book); err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "Invalid request body: " + err.Error(), + }) + return + } + + // Preserve the ID (in case it was in the JSON) + book.ID = uri.ID + + // Save to database + if err := utils.SaveBook(c.Request.Context(), book); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "Failed to update book: " + err.Error(), + }) + return + } + + // Sync to Typesense asynchronously (non-blocking) + go func(bookCopy models.Book) { + ctx := context.Background() + if err := utils.SyncBookOnUpdate(ctx, &bookCopy); err != nil { + log.Printf("Async Typesense sync failed for book %d: %v", bookCopy.ID, err) + } + }(*book) + + c.JSON(http.StatusOK, gin.H{ + "message": "Book updated successfully", + "book": book, + }) +} + +// deleteBook soft-deletes a book and removes it from Typesense +func deleteBook(c *gin.Context) { + var uri struct { + ID uint `uri:"id" binding:"required"` + } + + if err := c.ShouldBindUri(&uri); err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "Invalid book ID", + }) + return + } + + // Check if book exists + _, err := utils.GetBookByID(c.Request.Context(), uri.ID) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{ + "error": "Book not found", + }) + return + } + + // Soft delete from database + if err := utils.DeleteBook(c.Request.Context(), uri.ID); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "Failed to delete book: " + err.Error(), + }) + return + } + + // Remove from Typesense asynchronously (non-blocking) + go func(bookID uint) { + ctx := context.Background() + if err := utils.SyncBookDeletionOnDelete(ctx, bookID); err != nil { + log.Printf("Async Typesense deletion failed for book %d: %v", bookID, err) + } + }(uri.ID) + + c.JSON(http.StatusOK, gin.H{ + "message": "Book deleted successfully", + }) +} diff --git a/typesense-gin-full-text-search/routes/search.go b/typesense-gin-full-text-search/routes/search.go index dec7428..fc3e28f 100644 --- a/typesense-gin-full-text-search/routes/search.go +++ b/typesense-gin-full-text-search/routes/search.go @@ -2,6 +2,7 @@ package routes import ( "net/http" + "time" "github.com/gin-gonic/gin" "github.com/typesense/code-samples/typesense-gin-full-text-search/utils" @@ -13,6 +14,10 @@ import ( func SetupSearchRoutes(router *gin.Engine) { // Simple search endpoint router.GET("/search", searchBooks) + + // Sync endpoints for database-to-Typesense synchronization + router.POST("/sync", syncDatabaseToTypesense) + router.GET("/sync/status", getSyncStatus) } // searchBooks handles the search request @@ -56,3 +61,66 @@ func searchBooks(c *gin.Context) { "facet_counts": result.FacetCounts, }) } + +// syncDatabaseToTypesense triggers an immediate sync from database to Typesense +func syncDatabaseToTypesense(c *gin.Context) { + ctx := c.Request.Context() + + // Get last sync time from global state + lastSyncTime := utils.GetLastSyncTime() + + // Perform regular book sync (inserts and updates) + newSyncTime, err := utils.SyncBooksToTypesense(ctx, lastSyncTime) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "Sync failed", + "message": err.Error(), + }) + return + } + + // Handle soft deletes + deletedBooks, err := utils.GetDeletedBooks(ctx, lastSyncTime) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "Failed to fetch deleted books", + "message": err.Error(), + }) + return + } + + if len(deletedBooks) > 0 { + deletedIDs := make([]uint, 0, len(deletedBooks)) + for _, book := range deletedBooks { + deletedIDs = append(deletedIDs, book.ID) + } + + if err := utils.SyncSoftDeletesToTypesense(ctx, deletedIDs); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "Failed to sync deletions", + "message": err.Error(), + }) + return + } + } + + // Update global sync time + utils.SetLastSyncTime(newSyncTime) + + c.JSON(http.StatusOK, gin.H{ + "message": "Sync completed", + "newSyncTime": newSyncTime.Format(time.RFC3339), + "syncedAt": time.Now().Format(time.RFC3339), + "deletedBooks": len(deletedBooks), + }) +} + +// getSyncStatus returns the current sync status +func getSyncStatus(c *gin.Context) { + lastSyncTime := utils.GetLastSyncTime() + + c.JSON(http.StatusOK, gin.H{ + "lastSyncTime": lastSyncTime.Format(time.RFC3339), + "syncWorkerRunning": utils.IsSyncWorkerRunning(), + }) +} diff --git a/typesense-gin-full-text-search/server.go b/typesense-gin-full-text-search/server.go index b85c828..255bccd 100644 --- a/typesense-gin-full-text-search/server.go +++ b/typesense-gin-full-text-search/server.go @@ -12,6 +12,10 @@ import ( ) func main() { + + // Connect to database (stores global DB instance in utils.DB) + utils.ConnectToDB(context.Background()) + // Initialize collections before starting the server ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -21,19 +25,20 @@ func main() { } // Initialize data if collection is empty + // Use this if you want to import data from a JSONL file // This is idempotent - only imports if collection has no documents - dataFile := "books.jsonl" - if err := utils.InitializeDataIfEmpty(ctx, utils.BookCollection, dataFile); err != nil { - log.Printf("Warning: Failed to initialize data: %v", err) - log.Println("Server will continue, but collection may be empty") - } + // dataFile := "books.jsonl" + // if err := utils.InitializeDataIfEmpty(ctx, utils.BookCollection, dataFile); err != nil { + // log.Printf("Warning: Failed to initialize data: %v", err) + // log.Println("Server will continue, but collection may be empty") + // } router := gin.Default() // CORS middleware router.Use(cors.New(cors.Config{ AllowOrigins: []string{"*"}, - AllowMethods: []string{"GET", "OPTIONS"}, + AllowMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, AllowHeaders: []string{"Origin", "Content-Type", "Accept", "Authorization"}, ExposeHeaders: []string{"Content-Length"}, AllowCredentials: true, @@ -46,9 +51,19 @@ func main() { }) }) - // Setup search routes + // Setup search and sync routes routes.SetupSearchRoutes(router) + // Setup book CRUD routes + routes.SetupBookRoutes(router) + + // Start background sync worker + syncConfig := utils.DefaultSyncConfig() + syncConfig.EnableSoftDelete = true // Enable soft delete handling + go utils.StartSyncWorker(context.Background(), syncConfig) + port := utils.GetEnv("PORT", "3000") + log.Printf("Server starting on port %s", port) + log.Printf("Sync worker started with interval: %d seconds", syncConfig.SyncIntervalSec) router.Run(":" + port) } diff --git a/typesense-gin-full-text-search/utils/database.go b/typesense-gin-full-text-search/utils/database.go new file mode 100644 index 0000000..b48f93b --- /dev/null +++ b/typesense-gin-full-text-search/utils/database.go @@ -0,0 +1,140 @@ +package utils + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/typesense/code-samples/typesense-gin-full-text-search/models" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +var DB *gorm.DB + +func ConnectToDB(ctx context.Context) *gorm.DB { + host := os.Getenv("DB_HOST") + user := os.Getenv("DB_USER") + password := os.Getenv("DB_PASSWORD") + dbname := os.Getenv("DB_NAME") + port := os.Getenv("DB_PORT") + + dsn := fmt.Sprintf( + "host=%s user=%s password=%s dbname=%s port=%s sslmode=disable TimeZone=UTC", + host, user, password, dbname, port, + ) + + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) + if err != nil { + panic(fmt.Sprintf("Failed to connect to the database: %v", err)) + } + + // Auto-migrate the schema + if err := db.AutoMigrate(&models.Book{}); err != nil { + panic(fmt.Sprintf("Failed to auto-migrate: %v", err)) + } + + DB = db + return db +} + +// GetBookByID retrieves a single book by ID +func GetBookByID(ctx context.Context, id uint) (*models.Book, error) { + var book models.Book + if err := DB.WithContext(ctx).First(&book, id).Error; err != nil { + return nil, err + } + return &book, nil +} + +// GetBooksByUpdatedAt fetches books updated after a given time +// This is used for incremental sync to Typesense +func GetBooksByUpdatedAt(ctx context.Context, since time.Time) ([]models.Book, error) { + var books []models.Book + err := DB.WithContext(ctx). + Where("updated_at > ?", since). + Order("updated_at ASC"). + Find(&books).Error + return books, err +} + +// GetAllBooks fetches all books (for full import) +func GetAllBooks(ctx context.Context) ([]models.Book, error) { + var books []models.Book + err := DB.WithContext(ctx).Find(&books).Error + return books, err +} + +// GetDeletedBooks fetches soft-deleted books since a given time +// Uses updated_at to find books that were deleted (soft delete updates updated_at) +func GetDeletedBooks(ctx context.Context, since time.Time) ([]models.Book, error) { + var books []models.Book + err := DB.WithContext(ctx). + Unscoped(). // Include soft-deleted records + Where("deleted_at IS NOT NULL"). + Where("updated_at > ?", since). + Find(&books).Error + return books, err +} + +// SaveBook creates or updates a book +func SaveBook(ctx context.Context, book *models.Book) error { + return DB.WithContext(ctx).Save(book).Error +} + +// DeleteBook performs soft delete +func DeleteBook(ctx context.Context, id uint) error { + return DB.WithContext(ctx).Delete(&models.Book{}, id).Error +} + +// BulkSaveBooks inserts or updates multiple books using gORM's Create with bulk insert +// Note: For upsert (update on conflict), use database-specific syntax +func BulkSaveBooks(ctx context.Context, books []models.Book) error { + // Begin transaction + tx := DB.WithContext(ctx).Begin() + defer tx.Rollback() + + if err := tx.Create(books).Error; err != nil { + return err + } + + return tx.Commit().Error +} + +// BulkUpsertBooks performs PostgreSQL-specific upsert (ON CONFLICT) +// This is more efficient than gORM's default Create for updates +func BulkUpsertBooks(ctx context.Context, books []models.Book) error { + if len(books) == 0 { + return nil + } + + // Hardcode table name - gORM pluralizes model name + tableName := "books" + + // Build values string for INSERT + values := make([]string, 0, len(books)) + args := make([]any, 0, len(books)*7) + + for _, book := range books { + values = append(values, "(?, ?, ?, ?, ?, ?, ?)") + args = append(args, book.ID, book.Title, book.Authors, book.PublicationYear, book.AverageRating, book.ImageUrl, book.RatingsCount) + } + + // PostgreSQL ON CONFLICT query for upsert + // Updates all fields except id on conflict + query := fmt.Sprintf(` + INSERT INTO %s (id, title, authors, publication_year, average_rating, image_url, ratings_count) + VALUES %s + ON CONFLICT (id) DO UPDATE SET + title = EXCLUDED.title, + authors = EXCLUDED.authors, + publication_year = EXCLUDED.publication_year, + average_rating = EXCLUDED.average_rating, + image_url = EXCLUDED.image_url, + ratings_count = EXCLUDED.ratings_count, + updated_at = NOW() + `, tableName, joinStringSlice(values, ",")) + + return DB.WithContext(ctx).Exec(query, args...).Error +} diff --git a/typesense-gin-full-text-search/utils/sync.go b/typesense-gin-full-text-search/utils/sync.go new file mode 100644 index 0000000..9c138f2 --- /dev/null +++ b/typesense-gin-full-text-search/utils/sync.go @@ -0,0 +1,326 @@ +package utils + +import ( + "context" + "encoding/json" + "fmt" + "log" + "strings" + "sync" + "time" + + "github.com/typesense/typesense-go/v4/typesense/api" + "github.com/typesense/typesense-go/v4/typesense/api/pointer" + + "github.com/typesense/code-samples/typesense-gin-full-text-search/models" +) + +// SyncConfig holds configuration for the sync process +type SyncConfig struct { + BatchSize int + SyncIntervalSec int + EnableSoftDelete bool +} + +// DefaultSyncConfig returns default sync configuration +func DefaultSyncConfig() *SyncConfig { + return &SyncConfig{ + BatchSize: 100, + SyncIntervalSec: 60, // Sync every 60 seconds + } +} + +// SyncAllBooksToTypesense performs a full sync of all books from database to Typesense +// This should only be used for initial data load when Typesense is empty +// For regular syncing, use SyncBooksToTypesense which is incremental +func SyncAllBooksToTypesense(ctx context.Context) error { + log.Printf("Starting full sync of all books to Typesense...") + + // Get all books from DB + books, err := GetAllBooks(ctx) + if err != nil { + return fmt.Errorf("failed to fetch books from DB: %w", err) + } + + if len(books) == 0 { + log.Println("No books found in database") + return nil + } + + log.Printf("Syncing %d books to Typesense", len(books)) + + // Convert books to Typesense document format + documents := make([]any, 0, len(books)) + for _, book := range books { + doc := map[string]any{ + "id": book.GetTypesenseID(), + "title": book.Title, + "authors": book.Authors, + "publication_year": book.PublicationYear, + "average_rating": book.AverageRating, + "image_url": book.ImageUrl, + "ratings_count": book.RatingsCount, + } + documents = append(documents, doc) + } + + // Import documents in bulk using Typesense's import API + // Use upsert action to handle both inserts and updates + upsertAction := api.IndexAction("upsert") + importParams := &api.ImportDocumentsParams{ + BatchSize: pointer.Int(DefaultSyncConfig().BatchSize), + Action: &upsertAction, + } + + results, err := Client.Collection(BookCollection).Documents().Import( + ctx, + documents, + importParams, + ) + + if err != nil { + return fmt.Errorf("bulk import to Typesense failed: %w", err) + } + + // Count successes and failures + successCount := 0 + failureCount := 0 + for _, result := range results { + if result.Success { + successCount++ + } else { + failureCount++ + if failureCount <= 5 { + log.Printf("Sync error for document %s: %s", result.Id, result.Error) + } + } + } + + log.Printf("Full sync completed: %d documents upserted, %d failed", successCount, failureCount) + return nil +} + +// SyncBooksToTypesense fetches books changed since lastSyncTime and upserts them into Typesense +// This is an incremental sync - only syncs books modified since the last sync +// Returns the new lastSyncTime +func SyncBooksToTypesense(ctx context.Context, lastSyncTime time.Time) (time.Time, error) { + log.Printf("Starting incremental sync from database to Typesense since %s", lastSyncTime.Format(time.RFC3339)) + + // Get only books updated since last sync (efficient for large datasets) + books, err := GetBooksByUpdatedAt(ctx, lastSyncTime) + if err != nil { + return lastSyncTime, fmt.Errorf("failed to fetch changed books from DB: %w", err) + } + + if len(books) == 0 { + log.Println("No changes to sync") + return time.Now(), nil + } + + log.Printf("Found %d books to sync", len(books)) + + // Convert books to Typesense document format + documents := make([]any, 0, len(books)) + for _, book := range books { + doc := map[string]any{ + "id": book.GetTypesenseID(), + "title": book.Title, + "authors": book.Authors, + "publication_year": book.PublicationYear, + "average_rating": book.AverageRating, + "image_url": book.ImageUrl, + "ratings_count": book.RatingsCount, + } + documents = append(documents, doc) + } + + // Import documents in bulk using Typesense's import API + // Use upsert action to handle both inserts and updates + upsertAction := api.IndexAction("upsert") + importParams := &api.ImportDocumentsParams{ + BatchSize: pointer.Int(DefaultSyncConfig().BatchSize), + Action: &upsertAction, + } + + results, err := Client.Collection(BookCollection).Documents().Import( + ctx, + documents, + importParams, + ) + + if err != nil { + return lastSyncTime, fmt.Errorf("bulk import to Typesense failed: %w", err) + } + + // Count successes and failures + successCount := 0 + failureCount := 0 + for _, result := range results { + if result.Success { + successCount++ + } else { + failureCount++ + if failureCount <= 5 { + log.Printf("Sync error for document %s: %s", result.Id, result.Error) + } + } + } + + log.Printf("Sync completed: %d documents upserted, %d failed", successCount, failureCount) + + // Update last sync time + newSyncTime := time.Now() + log.Printf("Last sync time updated to: %s", newSyncTime.Format(time.RFC3339)) + + return newSyncTime, nil +} + +// SyncSoftDeletesToTypesense removes deleted books from Typesense +// Uses a filter query to delete multiple documents by ID +func SyncSoftDeletesToTypesense(ctx context.Context, deletedBookIDs []uint) error { + if len(deletedBookIDs) == 0 { + return nil + } + + // Convert IDs to Typesense document IDs (book_{ID}) + idStrings := make([]string, 0, len(deletedBookIDs)) + for _, id := range deletedBookIDs { + idStrings = append(idStrings, fmt.Sprintf("book_%d", id)) + } + + // Build filter: id:=[book_1,book_2,book_3] + filterBy := fmt.Sprintf("id:=[%s]", joinStringSlice(idStrings, ",")) + + log.Printf("Deleting %d documents from Typesense: %s", len(deletedBookIDs), filterBy) + + // Delete by query + _, err := Client.Collection(BookCollection).Documents().Delete(ctx, &api.DeleteDocumentsParams{ + FilterBy: pointer.String(filterBy), + }) + + if err != nil { + return fmt.Errorf("failed to delete documents from Typesense: %w", err) + } + + log.Printf("Successfully deleted %d documents from Typesense", len(deletedBookIDs)) + return nil +} + +// joinStringSlice joins string slice with separator using strings.Builder +func joinStringSlice(slice []string, sep string) string { + if len(slice) == 0 { + return "" + } + if len(slice) == 1 { + return slice[0] + } + var builder strings.Builder + builder.WriteString(slice[0]) + for i := 1; i < len(slice); i++ { + builder.WriteString(sep) + builder.WriteString(slice[i]) + } + return builder.String() +} + +// BookToDocument converts a Book model to a Typesense document map +func BookToDocument(book models.Book) map[string]any { + return map[string]any{ + "id": book.GetTypesenseID(), + "title": book.Title, + "authors": book.Authors, + "publication_year": book.PublicationYear, + "average_rating": book.AverageRating, + "image_url": book.ImageUrl, + "ratings_count": book.RatingsCount, + } +} + +// DocumentToBook converts a Typesense document map to a Book model +func DocumentToBook(doc map[string]any) (*models.Book, error) { + jsonBytes, err := json.Marshal(doc) + if err != nil { + return nil, err + } + + var book models.Book + if err := json.Unmarshal(jsonBytes, &book); err != nil { + return nil, err + } + + // Handle ID conversion from float64 to uint (JSON unmarshals numbers as float64) + if idFloat, ok := doc["id"].(float64); ok { + book.ID = uint(idFloat) + } + + return &book, nil +} + +// SyncSingleBookToTypesense updates a single book in Typesense (for real-time sync) +func SyncSingleBookToTypesense(ctx context.Context, book models.Book) error { + doc := BookToDocument(book) + + // Use the Upsert API for single document + _, err := Client.Collection(BookCollection).Documents().Upsert(ctx, doc, &api.DocumentIndexParameters{}) + if err != nil { + return fmt.Errorf("failed to upsert book to Typesense: %w", err) + } + + log.Printf("Synced single book to Typesense: ID=%d, Title=%s", book.ID, book.Title) + return nil +} + +// SyncSingleBookDeletionToTypesense deletes a single book from Typesense +func SyncSingleBookDeletionToTypesense(ctx context.Context, bookID uint) error { + // Delete by document ID (uses book_{ID} format) + documentID := fmt.Sprintf("book_%d", bookID) + + _, err := Client.Collection(BookCollection).Document(documentID).Delete(ctx) + if err != nil { + return fmt.Errorf("failed to delete book from Typesense: %w", err) + } + + log.Printf("Deleted book from Typesense: ID=%d", bookID) + return nil +} + +// SyncState tracks the current sync state +type SyncState struct { + LastSyncTime time.Time + SyncWorkerRunning bool + mu sync.RWMutex +} + +var ( + globalSyncState = &SyncState{ + LastSyncTime: time.Now(), + } +) + +// GetLastSyncTime returns the last sync time +func GetLastSyncTime() time.Time { + globalSyncState.mu.RLock() + defer globalSyncState.mu.RUnlock() + return globalSyncState.LastSyncTime +} + +// SetLastSyncTime updates the last sync time +func SetLastSyncTime(t time.Time) { + globalSyncState.mu.Lock() + defer globalSyncState.mu.Unlock() + globalSyncState.LastSyncTime = t +} + +// SetSyncWorkerRunning updates the sync worker status +func SetSyncWorkerRunning(running bool) { + globalSyncState.mu.Lock() + defer globalSyncState.mu.Unlock() + globalSyncState.SyncWorkerRunning = running +} + +// IsSyncWorkerRunning returns whether the sync worker is running +func IsSyncWorkerRunning() bool { + globalSyncState.mu.RLock() + defer globalSyncState.mu.RUnlock() + return globalSyncState.SyncWorkerRunning +} diff --git a/typesense-gin-full-text-search/utils/sync_worker.go b/typesense-gin-full-text-search/utils/sync_worker.go new file mode 100644 index 0000000..9d9e5d9 --- /dev/null +++ b/typesense-gin-full-text-search/utils/sync_worker.go @@ -0,0 +1,133 @@ +package utils + +import ( + "context" + "log" + "time" + + "github.com/typesense/code-samples/typesense-gin-full-text-search/models" +) + +var ( + workerCtx context.Context + workerCancel context.CancelFunc +) + +// StartSyncWorker starts a background worker that periodically syncs database changes to Typesense +func StartSyncWorker(ctx context.Context, config *SyncConfig) { + workerCtx, workerCancel = context.WithCancel(ctx) + SetSyncWorkerRunning(true) + + log.Printf("Starting sync worker with interval: %d seconds", config.SyncIntervalSec) + + // Initial sync + go func() { + // Wait a bit before first sync to allow server to start + time.Sleep(2 * time.Second) + lastSyncTime := GetLastSyncTime() + if _, err := SyncBooksToTypesense(workerCtx, lastSyncTime); err != nil { + log.Printf("Initial sync failed: %v", err) + } + }() + + // Periodic sync loop + ticker := time.NewTicker(time.Duration(config.SyncIntervalSec) * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + log.Printf("Running periodic sync...") + lastSyncTime := GetLastSyncTime() + if _, err := SyncBooksToTypesense(workerCtx, lastSyncTime); err != nil { + log.Printf("Periodic sync failed: %v", err) + } + // Handle soft deletes if enabled + if config.EnableSoftDelete { + if err := handleSoftDeletes(workerCtx); err != nil { + log.Printf("Soft delete sync failed: %v", err) + } + } + case <-workerCtx.Done(): + log.Println("Sync worker stopped") + SetSyncWorkerRunning(false) + return + } + } +} + +// StopSyncWorker stops the background sync worker +func StopSyncWorker() { + if workerCancel != nil { + workerCancel() + } +} + +// handleSoftDeletes processes soft-deleted books and removes them from Typesense +func handleSoftDeletes(ctx context.Context) error { + lastSyncTime := GetLastSyncTime() + + log.Printf("handleSoftDeletes: checking for soft-deleted books since %s", lastSyncTime.Format(time.RFC3339)) + + // Get soft-deleted books since last sync + deletedBooks, err := GetDeletedBooks(ctx, lastSyncTime) + if err != nil { + return err + } + + log.Printf("handleSoftDeletes: found %d soft-deleted books", len(deletedBooks)) + + if len(deletedBooks) == 0 { + return nil + } + + // Collect IDs of deleted books + deletedIDs := make([]uint, 0, len(deletedBooks)) + for _, book := range deletedBooks { + deletedIDs = append(deletedIDs, book.ID) + } + + log.Printf("Found %d soft-deleted books to sync to Typesense", len(deletedIDs)) + + // Sync deletions to Typesense + if err := SyncSoftDeletesToTypesense(ctx, deletedIDs); err != nil { + return err + } + + // Clear soft deletes from database (optional - depends on your retention policy) + // Note: gORM's Delete does soft delete if model has DeletedAt field + // To permanently delete, use Unscoped(): + // DB.Unscoped().Where("id IN ?", deletedIDs).Delete(&models.Book{}) + + // Update last sync time + SetLastSyncTime(time.Now()) + + return nil +} + +// SyncBookOnUpdate handles real-time sync when a book is created or updated +// This is meant to be called from your Gin handlers after DB operations +func SyncBookOnUpdate(ctx context.Context, book *models.Book) error { + // Sync to Typesense immediately (real-time) + if err := SyncSingleBookToTypesense(ctx, *book); err != nil { + return err + } + + // Update sync state timestamp + SetLastSyncTime(time.Now()) + + return nil +} + +// SyncBookDeletionOnDelete handles real-time sync when a book is deleted +func SyncBookDeletionOnDelete(ctx context.Context, bookID uint) error { + // Sync deletion to Typesense immediately (real-time) + if err := SyncSingleBookDeletionToTypesense(ctx, bookID); err != nil { + return err + } + + // Update sync state timestamp + SetLastSyncTime(time.Now()) + + return nil +} From 5695a3c6c047c2ab013b85e414cdfd7aa261fa8d Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Wed, 25 Feb 2026 08:11:58 +0530 Subject: [PATCH 4/5] Third revision: Add comprehensive documentation for PostgreSQL integration, sync strategies, and deployment guide --- typesense-gin-full-text-search/README.md | 296 +++++++++++++++++++++-- 1 file changed, 273 insertions(+), 23 deletions(-) diff --git a/typesense-gin-full-text-search/README.md b/typesense-gin-full-text-search/README.md index c256d50..86d8733 100644 --- a/typesense-gin-full-text-search/README.md +++ b/typesense-gin-full-text-search/README.md @@ -1,18 +1,19 @@ # Gin Full-Text Search with Typesense -A RESTful search API built with Go Gin framework and Typesense, featuring full-text search capabilities with environment-based configuration. +A production-ready RESTful search API built with Go Gin framework, PostgreSQL, and Typesense. Features full-text search, CRUD operations, real-time async indexing, and background sync workers. ## Tech Stack - Go 1.19+ - Gin Web Framework +- PostgreSQL with GORM - Typesense - +- Docker ## Prerequisites - Go 1.19+ installed -- Docker (for running Typesense locally). Alternatively, you can use a Typesense Cloud cluster. -- Basic knowledge of Go and REST APIs. +- Docker and Docker Compose (for Typesense and PostgreSQL) +- Basic knowledge of Go, REST APIs, and SQL ## Quick Start @@ -29,31 +30,68 @@ cd typesense-gin-full-text-search go mod download ``` -### 3. Set up environment variables +### 3. Start Typesense and PostgreSQL + +Run Typesense and PostgreSQL using Docker: + +```bash +# Start Typesense +docker run -d \ + -p 8108:8108 \ + -v typesense-data:/data \ + typesense/typesense:30.1 \ + --data-dir /data \ + --api-key=1234 \ + --enable-cors + +# Start PostgreSQL +docker run -d \ + -p 5432:5432 \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=password \ + -e POSTGRES_DB=typesense_books \ + -v postgres-data:/var/lib/postgresql/data \ + postgres:15 +``` + +### 4. Set up environment variables -Create a `.env` file in the project root with the following content: +Create a `.env` file in the project root: ```env # Server Configuration PORT=3000 +# Database Configuration +DB_HOST=localhost +DB_USER=postgres +DB_PASSWORD=password +DB_NAME=typesense_books +DB_PORT=5432 + # Typesense Configuration TYPESENSE_HOST=localhost TYPESENSE_PORT=8108 TYPESENSE_PROTOCOL=http TYPESENSE_API_KEY=xyz -TYPESENSE_COLLECTION=books ``` ### 4. Project Structure ```text -├── routes -│ └── search.go -├── utils -│ ├── env.go -│ └── typesense.go -├── server.go +├── models/ +│ └── book.go # Book model with GORM tags +├── routes/ +│ ├── books.go # CRUD endpoints for books +│ └── search.go # Search and sync endpoints +├── utils/ +│ ├── collections.go # Typesense collection schema +│ ├── database.go # PostgreSQL database operations +│ ├── env.go # Environment variable helpers +│ ├── sync.go # Sync logic (incremental, full, soft delete) +│ ├── sync_worker.go # Background sync worker +│ └── typesense.go # Typesense client initialization +├── server.go # Main application entry point ├── go.mod └── .env ``` @@ -84,9 +122,9 @@ The server will automatically restart when you make changes to any Go file. Open [http://localhost:3000](http://localhost:3000) in your browser. -### 6. Search API Endpoint +### 7. API Endpoints -**Search:** +#### Search ```bash GET /search?q= @@ -98,23 +136,235 @@ Example: curl "http://localhost:3000/search?q=harry" ``` -### 7. Deployment +Response: + +```json +{ + "query": "harry", + "found": 7, + "results": [...], + "facet_counts": [...] +} +``` + +#### CRUD Operations + +**Create a book:** + +```bash +POST /books +Content-Type: application/json + +{ + "title": "The Go Programming Language", + "authors": ["Alan Donovan", "Brian Kernighan"], + "publication_year": 2015, + "average_rating": 4.5, + "image_url": "https://example.com/image.jpg", + "ratings_count": 1000 +} +``` + +**Get a book:** + +```bash +GET /books/:id +``` + +**Get all books:** + +```bash +GET /books +``` + +**Update a book:** + +```bash +PUT /books/:id +Content-Type: application/json + +{ + "title": "Updated Title", + "authors": ["Author Name"], + "publication_year": 2024, + "average_rating": 4.8, + "image_url": "https://example.com/updated.jpg", + "ratings_count": 1500 +} +``` + +**Delete a book (soft delete):** + +```bash +DELETE /books/:id +``` + +#### Sync Operations + +**Trigger manual sync:** + +```bash +POST /sync +``` + +Response: + +```json +{ + "message": "Sync completed", + "newSyncTime": "2026-02-25T07:54:11+05:30", + "syncedAt": "2026-02-25T07:54:11+05:30", + "deletedBooks": 1 +} +``` + +**Check sync status:** + +```bash +GET /sync/status +``` + +Response: + +```json +{ + "lastSyncTime": "2026-02-25T07:54:11+05:30", + "syncWorkerRunning": true +} +``` -Set env variables to point the app to the Typesense Cluster: +### 8. How It Works + +#### Architecture + +``` +User Request + ↓ +Gin API (CRUD) + ↓ +PostgreSQL (Source of Truth) + ↓ +Async Sync → Typesense (Search Index) + ↑ +Background Worker (Every 60s) +``` + +#### Sync Strategies + +**1. Real-time Sync (Async)** +- Triggered on: Create, Update, Delete operations +- Non-blocking: API responds immediately +- Runs in background goroutine +- If fails: Background worker catches it within 60 seconds + +**2. Background Periodic Sync** +- Runs every 60 seconds automatically +- Incremental: Only syncs books with `updated_at > lastSyncTime` +- Handles soft deletes: Removes deleted books from Typesense +- Efficient: Uses upsert to handle both inserts and updates + +**3. Manual Sync** +- Endpoint: `POST /sync` +- On-demand sync trigger +- Useful for debugging or forced sync + +#### Database Schema + +```sql +CREATE TABLE books ( + id SERIAL PRIMARY KEY, + title VARCHAR(255), + authors JSONB, -- Array stored as JSON + publication_year INTEGER, + average_rating DECIMAL, + image_url VARCHAR(255), + ratings_count INTEGER, + created_at TIMESTAMP, + updated_at TIMESTAMP, + deleted_at TIMESTAMP -- Soft delete support +); +``` + +#### Typesense Collection Schema + +```go +{ + "name": "books", + "fields": [ + {"name": "id", "type": "string"}, + {"name": "title", "type": "string"}, + {"name": "authors", "type": "string[]", "facet": true}, + {"name": "publication_year", "type": "int32", "facet": true}, + {"name": "average_rating", "type": "float", "facet": true}, + {"name": "image_url", "type": "string"}, + {"name": "ratings_count", "type": "int32"} + ] +} +``` + +### 9. Deployment + +**Environment Variables for Production:** ```env # Server Configuration PORT=3000 +GIN_MODE=release -# Typesense Configuration +# Database Configuration (use managed PostgreSQL) +DB_HOST=your-postgres-host.com +DB_USER=your-db-user +DB_PASSWORD=your-secure-password +DB_NAME=typesense_books +DB_PORT=5432 + +# Typesense Configuration (use Typesense Cloud) TYPESENSE_HOST=xxx.typesense.net TYPESENSE_PORT=443 TYPESENSE_PROTOCOL=https TYPESENSE_API_KEY=your-production-api-key -TYPESENSE_COLLECTION=books ``` -- Configure CORS middleware for specific origins. -- Configure gin to run in release mode. -- Add some sort of authentication to the API. -- Add rate limiting to the API. +### 10. Testing the Sync + +**Test real-time sync:** + +```bash +# 1. Create a book via API +curl -X POST http://localhost:3000/books \ + -H "Content-Type: application/json" \ + -d '{"title": "Test Book", "authors": ["Author"], "publication_year": 2024}' + +# 2. Search immediately (should appear) +curl "http://localhost:3000/search?q=Test" +``` + +**Test background sync:** + +```bash +# 1. Insert book directly in database (bypassing API) +psql -h localhost -U postgres -d typesense_books -c " +INSERT INTO books (title, authors, publication_year, created_at, updated_at) +VALUES ('Direct DB Book', '[\"DB Author\"]', 2025, NOW(), NOW()); +" + +# 2. Wait 60 seconds for background worker + +# 3. Search (should appear after sync) +curl "http://localhost:3000/search?q=Direct" +``` + +**Test soft delete sync:** + +```bash +# 1. Soft delete a book in database +psql -h localhost -U postgres -d typesense_books -c " +UPDATE books SET deleted_at = NOW(), updated_at = NOW() WHERE id = 1; +" + +# 2. Trigger manual sync or wait 60 seconds +curl -X POST http://localhost:3000/sync + +# 3. Search (should not appear) +curl "http://localhost:3000/search?q=" +``` From 34ed409e184e01ef72761547f45c280a46f1b6e0 Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Wed, 25 Feb 2026 08:14:48 +0530 Subject: [PATCH 5/5] Fourth revision: Update README formatting and clarify Docker prerequisites --- typesense-gin-full-text-search/README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/typesense-gin-full-text-search/README.md b/typesense-gin-full-text-search/README.md index 86d8733..83612c6 100644 --- a/typesense-gin-full-text-search/README.md +++ b/typesense-gin-full-text-search/README.md @@ -9,10 +9,11 @@ A production-ready RESTful search API built with Go Gin framework, PostgreSQL, a - PostgreSQL with GORM - Typesense - Docker + ## Prerequisites - Go 1.19+ installed -- Docker and Docker Compose (for Typesense and PostgreSQL) +- Docker (for Typesense and PostgreSQL) - Basic knowledge of Go, REST APIs, and SQL ## Quick Start @@ -237,7 +238,7 @@ Response: #### Architecture -``` +```plaintext User Request ↓ Gin API (CRUD) @@ -251,19 +252,22 @@ Background Worker (Every 60s) #### Sync Strategies -**1. Real-time Sync (Async)** +##### 1. Real-time Sync (Async) + - Triggered on: Create, Update, Delete operations - Non-blocking: API responds immediately - Runs in background goroutine - If fails: Background worker catches it within 60 seconds -**2. Background Periodic Sync** +##### 2. Background Periodic Sync + - Runs every 60 seconds automatically - Incremental: Only syncs books with `updated_at > lastSyncTime` - Handles soft deletes: Removes deleted books from Typesense - Efficient: Uses upsert to handle both inserts and updates -**3. Manual Sync** +##### 3. Manual Sync + - Endpoint: `POST /sync` - On-demand sync trigger - Useful for debugging or forced sync