Core 2637/data loader

config.yaml

@@ -12,4 +12,7 @@ static: # This allows static files to be directly accessible
   root: web
   files: web/**
 roles: # This can define the roles that are used in the application
-  files: roles.yaml
+  files: roles.yaml
+dataLoader: # This loads data into user tables from YAML or JSON files
+  files: data/*.{json,yaml,yml}
+  # forceLoad: true  # Uncomment to force loading data even if tables already have records

data/README.md

@@ -0,0 +1,73 @@
+# HarperDB Data Loader
+
+This directory contains YAML or JSON files that are automatically loaded into your HarperDB database tables when your application starts.
+
+## How It Works
+
+1. Place data files in this directory with `.yaml`, `.yml`, or `.json` extensions
+2. Files are processed when HarperDB starts
+3. Records are inserted/updated based on file modification time:
+   - New records are always added
+   - Existing records are only updated if the file's modification time is newer than the record's stored timestamp
+   - Records with timestamps newer than the file's modification time are preserved unchanged
+
+## File Format
+
+Data files must contain `table` and `records` fields. The `database` field is optional.
+
+### JSON Example
+
+```json
+{
+  "database": "dev",       // Optional - uses default database if omitted
+  "table": "Product",      // Required - name of the table
+  "records": [             // Required - array of records
+    {
+      "id": "1",           // Primary key field
+      "name": "Laptop",
+      "price": 999.99
+    }
+  ]
+}
+```
+
+### YAML Example
+
+```yaml
+database: dev              # Optional - uses default database if omitted
+table: products            # Required - name of the table
+records:                   # Required - array of records
+  - id: 1                  # Primary key field
+    name: "Laptop"
+    price: 999.99
+```
+
+## Key Features
+
+- **Automatic Table Creation**: Tables are created if they don't exist
+- **Primary Key Detection**: The `id` field is automatically detected as the primary key
+- **File Modification Time**:
+  - The data loader uses the file's modification time (`mtime`) to determine if records should be updated
+  - "Touching" a file (updating its modification time) will force a reload of its data
+  - This allows for simpler data files without timestamp properties in the records
+- **Multi-file Support**: You can have multiple data files for different tables
+- **Complex Data Types**: Supports nested objects, arrays, and various data types
+- **One Table Per File**: Each file should define one table
+
+## Tips for Managing Data
+
+- To force a reload of data, simply update the file's modification time:
+  ```bash
+  # Update the file's timestamp using touch
+  touch data/products.json
+  ```
+- If you need to restore to previous data, you can replace the file and update its timestamp
+- The system automatically handles the comparison between file modification time and record timestamps
+
+## Sample Files
+
+- `categories.json`: Category data with parent/child relationships
+- `products.json`: Product data with references to categories
+- `users.json`: User account data
+
+These sample files demonstrate common data patterns and relationships.

data/categories.json

@@ -0,0 +1,26 @@
+{
+  "table": "Category",
+  "records": [
+    {
+      "id": "electronics",
+      "name": "Electronics",
+      "description": "Electronic devices and gadgets. LFG!!1"
+    },
+    {
+      "id": "furniture",
+      "name": "Furniture",
+      "description": "Home and office furniture"
+    },
+    {
+      "id": "clothing",
+      "name": "Clothing",
+      "description": "Apparel and accessories"
+    },
+    {
+      "id": "pants",
+      "name": "Pants",
+      "description": "Apparel and accessories",
+      "categoryId": "clothing"
+    }
+  ]
+}

data/products.json

@@ -0,0 +1,37 @@
+{
+  "table": "Product",
+  "records": [
+    {
+      "id": "1",
+      "name": "Laptop",
+      "price": 999.99,
+      "categoryId": "electronics",
+      "inStock": true
+    },
+    {
+      "id": "2",
+      "name": "Smartphone",
+      "price": 699.99,
+      "categoryId": "electronics",
+      "inStock": false
+    },
+    {
+      "id": "3",
+      "name": "Desk Chair",
+      "price": 199.99,
+      "categoryId": "furniture",
+      "details": {
+        "weight": 1.5,
+        "color": "black"
+      },
+      "inStock": true
+    },
+    {
+      "id": "4",
+      "name": "Jeans",
+      "price": 98.99,
+      "categoryId": "pants",
+      "inStock": true
+    }
+  ]
+}

data/sample-data.yaml

@@ -0,0 +1,63 @@
+# This is a sample YAML data file for HarperDB Data Loader
+# Format: { database, table, records[] }
+# 
+# This example shows inserting tables and records into a 
+# specific database, instead of the default 'data' database.
+# 
+# Example: Products for an e-commerce application
+# ```yaml
+# database: dev
+# table: products
+# records:
+#   - id: 1
+#     name: "Laptop"
+#     price: 999.99
+#     category: "Electronics"
+#     inStock: true
+#     __createdtime__: 1682752800000
+#     __updatedtime__: 1682752800000
+#   - id: 2
+#     name: "Smartphone"
+#     price: 699.99
+#     category: "Electronics"
+#     inStock: false
+#     __createdtime__: 1682752801000
+#     __updatedtime__: 1682752801000
+#   - id: 3
+#     name: "Desk Chair"
+#     price: 199.99
+#     category: "Furniture"
+#     inStock: true
+#     __createdtime__: 1682752802000
+#     __updatedtime__: 1682752802000
+# ```
+# 
+# If you need to load data into multiple tables, use separate files
+# for each table. For example:
+#
+# categories.yaml:
+# ```yaml
+# database: dev
+# table: categories
+# records:
+#   - id: "electronics"
+#     name: "Electronics"
+#     description: "Electronic devices and gadgets"
+#     __createdtime__: 1682752803000
+#     __updatedtime__: 1682752803000
+# ```
+#
+# users.yaml:
+# ```yaml
+# database: dev
+# table: users
+# records:
+#   - id: 1
+#     username: "john_doe"
+#     email: "john@example.com"
+#     firstName: "John"
+#     lastName: "Doe"
+#     active: true
+#     __createdtime__: 1682752806000
+#     __updatedtime__: 1682752806000
+# ```

data/users-myco.json

@@ -0,0 +1,20 @@
+{
+  "database": "myco",
+  "table": "User",
+  "records": [
+    {
+      "id": "1",
+      "username": "jane_doe",
+      "email": "jane@example.com",
+      "firstName": "Jane",
+      "lastName": "Doe"
+    },
+    {
+      "id": "2",
+      "username": "Jon_smith",
+      "email": "jon@example.com",
+      "firstName": "Jon",
+      "lastName": "Smith"
+    }
+  ]
+}

data/users.json

@@ -0,0 +1,19 @@
+{
+  "table": "User",
+  "records": [
+    {
+      "id": "1",
+      "username": "john_doe",
+      "email": "john@example.com",
+      "firstName": "John",
+      "lastName": "Doe"
+    },
+    {
+      "id": "2",
+      "username": "jane_smith",
+      "email": "jane@example.com",
+      "firstName": "Jane",
+      "lastName": "Smith"
+    }
+  ]
+}

schema.graphql

@@ -1,7 +1,45 @@
-## Here we can define any tables in our database. This example shows how we define a type as a table using
-## the type name as the table name and specifying it is an "export" available in the REST and other external protocols.
+## This schema defines the data model for our application.
+## Each type with the @table directive becomes a table in the database.
+## The @export directive makes it available in the REST API and other external protocols.
+
+## Example table for reference
 type TableName @table @export {
     id: ID @primaryKey # Here we define primary key (must be one)
     name: String # we can define any other attributes here
     tag: String @indexed # we can specify any attributes that should be indexed
 }
+
+## Example tables for data loader example
+type Product @table @export {
+    id: ID @primaryKey
+    name: String @indexed
+    price: Float
+    category: Category @relationship(from: "categoryId")
+    details: Any
+    inStock: Boolean
+}
+
+type Category @table @export {
+    id: ID @primaryKey
+    name: String @indexed
+    description: String
+    parent: Category @relationship(from: "categoryId")
+    products: [Product] @relationship(to: "categoryId")
+    children: [Category] @relationship(to: "categoryId")
+}
+
+type User @table @export {
+    id: ID @primaryKey
+    username: String @indexed
+    email: String @indexed
+    firstName: String
+    lastName: String
+}
+
+type MycoUser @table(database: "myco", table: "User") @export {
+    id: ID @primaryKey
+    username: String @indexed
+    email: String @indexed
+    firstName: String
+    lastName: String
+}