diff --git a/config.yaml b/config.yaml index 04bebfd..29d728c 100644 --- a/config.yaml +++ b/config.yaml @@ -12,4 +12,7 @@ static: # This allows static files to be directly accessible root: web files: web/** roles: # This can define the roles that are used in the application - files: roles.yaml \ No newline at end of file + files: roles.yaml +dataLoader: # This loads data into user tables from YAML or JSON files + files: data/*.{json,yaml,yml} + # forceLoad: true # Uncomment to force loading data even if tables already have records \ No newline at end of file diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..b7f8ef3 --- /dev/null +++ b/data/README.md @@ -0,0 +1,73 @@ +# HarperDB Data Loader + +This directory contains YAML or JSON files that are automatically loaded into your HarperDB database tables when your application starts. + +## How It Works + +1. Place data files in this directory with `.yaml`, `.yml`, or `.json` extensions +2. Files are processed when HarperDB starts +3. Records are inserted/updated based on file modification time: + - New records are always added + - Existing records are only updated if the file's modification time is newer than the record's stored timestamp + - Records with timestamps newer than the file's modification time are preserved unchanged + +## File Format + +Data files must contain `table` and `records` fields. The `database` field is optional. + +### JSON Example + +```json +{ + "database": "dev", // Optional - uses default database if omitted + "table": "Product", // Required - name of the table + "records": [ // Required - array of records + { + "id": "1", // Primary key field + "name": "Laptop", + "price": 999.99 + } + ] +} +``` + +### YAML Example + +```yaml +database: dev # Optional - uses default database if omitted +table: products # Required - name of the table +records: # Required - array of records + - id: 1 # Primary key field + name: "Laptop" + price: 999.99 +``` + +## Key Features + +- **Automatic Table Creation**: Tables are created if they don't exist +- **Primary Key Detection**: The `id` field is automatically detected as the primary key +- **File Modification Time**: + - The data loader uses the file's modification time (`mtime`) to determine if records should be updated + - "Touching" a file (updating its modification time) will force a reload of its data + - This allows for simpler data files without timestamp properties in the records +- **Multi-file Support**: You can have multiple data files for different tables +- **Complex Data Types**: Supports nested objects, arrays, and various data types +- **One Table Per File**: Each file should define one table + +## Tips for Managing Data + +- To force a reload of data, simply update the file's modification time: + ```bash + # Update the file's timestamp using touch + touch data/products.json + ``` +- If you need to restore to previous data, you can replace the file and update its timestamp +- The system automatically handles the comparison between file modification time and record timestamps + +## Sample Files + +- `categories.json`: Category data with parent/child relationships +- `products.json`: Product data with references to categories +- `users.json`: User account data + +These sample files demonstrate common data patterns and relationships. \ No newline at end of file diff --git a/data/categories.json b/data/categories.json new file mode 100644 index 0000000..3c387fb --- /dev/null +++ b/data/categories.json @@ -0,0 +1,26 @@ +{ + "table": "Category", + "records": [ + { + "id": "electronics", + "name": "Electronics", + "description": "Electronic devices and gadgets. LFG!!1" + }, + { + "id": "furniture", + "name": "Furniture", + "description": "Home and office furniture" + }, + { + "id": "clothing", + "name": "Clothing", + "description": "Apparel and accessories" + }, + { + "id": "pants", + "name": "Pants", + "description": "Apparel and accessories", + "categoryId": "clothing" + } + ] +} \ No newline at end of file diff --git a/data/products.json b/data/products.json new file mode 100644 index 0000000..cbe9416 --- /dev/null +++ b/data/products.json @@ -0,0 +1,37 @@ +{ + "table": "Product", + "records": [ + { + "id": "1", + "name": "Laptop", + "price": 999.99, + "categoryId": "electronics", + "inStock": true + }, + { + "id": "2", + "name": "Smartphone", + "price": 699.99, + "categoryId": "electronics", + "inStock": false + }, + { + "id": "3", + "name": "Desk Chair", + "price": 199.99, + "categoryId": "furniture", + "details": { + "weight": 1.5, + "color": "black" + }, + "inStock": true + }, + { + "id": "4", + "name": "Jeans", + "price": 98.99, + "categoryId": "pants", + "inStock": true + } + ] +} \ No newline at end of file diff --git a/data/sample-data.yaml b/data/sample-data.yaml new file mode 100644 index 0000000..5a8f03c --- /dev/null +++ b/data/sample-data.yaml @@ -0,0 +1,63 @@ +# This is a sample YAML data file for HarperDB Data Loader +# Format: { database, table, records[] } +# +# This example shows inserting tables and records into a +# specific database, instead of the default 'data' database. +# +# Example: Products for an e-commerce application +# ```yaml +# database: dev +# table: products +# records: +# - id: 1 +# name: "Laptop" +# price: 999.99 +# category: "Electronics" +# inStock: true +# __createdtime__: 1682752800000 +# __updatedtime__: 1682752800000 +# - id: 2 +# name: "Smartphone" +# price: 699.99 +# category: "Electronics" +# inStock: false +# __createdtime__: 1682752801000 +# __updatedtime__: 1682752801000 +# - id: 3 +# name: "Desk Chair" +# price: 199.99 +# category: "Furniture" +# inStock: true +# __createdtime__: 1682752802000 +# __updatedtime__: 1682752802000 +# ``` +# +# If you need to load data into multiple tables, use separate files +# for each table. For example: +# +# categories.yaml: +# ```yaml +# database: dev +# table: categories +# records: +# - id: "electronics" +# name: "Electronics" +# description: "Electronic devices and gadgets" +# __createdtime__: 1682752803000 +# __updatedtime__: 1682752803000 +# ``` +# +# users.yaml: +# ```yaml +# database: dev +# table: users +# records: +# - id: 1 +# username: "john_doe" +# email: "john@example.com" +# firstName: "John" +# lastName: "Doe" +# active: true +# __createdtime__: 1682752806000 +# __updatedtime__: 1682752806000 +# ``` \ No newline at end of file diff --git a/data/users-myco.json b/data/users-myco.json new file mode 100644 index 0000000..c824379 --- /dev/null +++ b/data/users-myco.json @@ -0,0 +1,20 @@ +{ + "database": "myco", + "table": "User", + "records": [ + { + "id": "1", + "username": "jane_doe", + "email": "jane@example.com", + "firstName": "Jane", + "lastName": "Doe" + }, + { + "id": "2", + "username": "Jon_smith", + "email": "jon@example.com", + "firstName": "Jon", + "lastName": "Smith" + } + ] +} \ No newline at end of file diff --git a/data/users.json b/data/users.json new file mode 100644 index 0000000..613b012 --- /dev/null +++ b/data/users.json @@ -0,0 +1,19 @@ +{ + "table": "User", + "records": [ + { + "id": "1", + "username": "john_doe", + "email": "john@example.com", + "firstName": "John", + "lastName": "Doe" + }, + { + "id": "2", + "username": "jane_smith", + "email": "jane@example.com", + "firstName": "Jane", + "lastName": "Smith" + } + ] +} \ No newline at end of file diff --git a/schema.graphql b/schema.graphql index 5ad5d48..b346d16 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1,7 +1,45 @@ -## Here we can define any tables in our database. This example shows how we define a type as a table using -## the type name as the table name and specifying it is an "export" available in the REST and other external protocols. +## This schema defines the data model for our application. +## Each type with the @table directive becomes a table in the database. +## The @export directive makes it available in the REST API and other external protocols. + +## Example table for reference type TableName @table @export { id: ID @primaryKey # Here we define primary key (must be one) name: String # we can define any other attributes here tag: String @indexed # we can specify any attributes that should be indexed } + +## Example tables for data loader example +type Product @table @export { + id: ID @primaryKey + name: String @indexed + price: Float + category: Category @relationship(from: "categoryId") + details: Any + inStock: Boolean +} + +type Category @table @export { + id: ID @primaryKey + name: String @indexed + description: String + parent: Category @relationship(from: "categoryId") + products: [Product] @relationship(to: "categoryId") + children: [Category] @relationship(to: "categoryId") +} + +type User @table @export { + id: ID @primaryKey + username: String @indexed + email: String @indexed + firstName: String + lastName: String +} + +type MycoUser @table(database: "myco", table: "User") @export { + id: ID @primaryKey + username: String @indexed + email: String @indexed + firstName: String + lastName: String +} \ No newline at end of file