diff --git a/bolt/README.md b/bolt/README.md index 5e27c40..c129b64 100644 --- a/bolt/README.md +++ b/bolt/README.md @@ -50,6 +50,13 @@ Usage: Copy a bolt package to a remote device via SSH and optionally install it via middleware --direct Skip middleware installation and deploy directly to the bolt packages directory + bolt fetch [--force] + Download a bolt package from the configured package store server into the local package store + can be a package name (id+version) or file name (id+version.bolt) + --force Replace the package if it already exists in the local package store + Package store URL, type and credentials are configured in ~/.bolt/config.json + See https://github.com/rdkcentral/bolt-tools/blob/main/bolt/docs/fetch.md + bolt run [options] Execute a bolt package on a remote device. If a package ID is provided (without version), the package is always launched via middleware. @@ -89,6 +96,7 @@ Global options (can be used with any command): --verbose Print detailed output during execution ``` +A detailed description of the `bolt fetch` command can be found in the [docs/fetch.md](docs/fetch.md) file. A detailed description of the `bolt make` command can be found in the [docs/make.md](docs/make.md) file. A detailed description of the `bolt push` command can be found in the [docs/push.md](docs/push.md) file. A detailed description of the `bolt run` command can be found in the [docs/run.md](docs/run.md) file. @@ -121,6 +129,7 @@ Supported options: |--------|--------------------------------------------------| | `key` | Default path to the RSA private key (PEM format) | | `cert` | Default path to the X.509 certificate (PEM format) | +| `packageStore*` | Package store settings used by `bolt fetch` (`packageStoreURL`, `packageStoreType`, etc.). See [docs/fetch.md](docs/fetch.md) | Example `~/.bolt/config.json`: ```json @@ -135,7 +144,8 @@ so the example above can be simplified to: ```json { "key": "signing.key.pem", - "cert": "signing.cert.pem" + "cert": "signing.cert.pem", + "packageStoreURL": "https://packages.example.com/bolts" } ``` diff --git a/bolt/docs/fetch.md b/bolt/docs/fetch.md new file mode 100644 index 0000000..de36776 --- /dev/null +++ b/bolt/docs/fetch.md @@ -0,0 +1,106 @@ +# bolt fetch Command Overview + +## Purpose + +The `bolt fetch` command downloads a bolt package from a remote package store server into the +[local package store](local-package-store.md). + +## Usage + +``` +bolt fetch [--force] +``` + +- `` is the package name (`id+version`) or file name (`id+version.bolt`). + It may be prefixed with a sub-directory path (for example an architecture, `arm/`), + which is appended to the server URL when downloading. + +## Options + +| Option | Description | +|-------------|--------------------------------------------------------------------------------------| +| --force | Replace the package if it already exists in the local package store. | + +## Configuration + +The remote server URL and package store type are configured in `~/.bolt/config.json`: + +```json +{ + "packageStoreURL": "https://packages.example.com/bolts" +} +``` + +| Key | Description | +|--------------------|--------------------------------------------------------------------| +| `packageStoreURL` | URL of the remote package store server (required) | +| `packageStoreType` | Package store type: `"basic"` (default) or `"rdk"` (see below) | +| `packageStoreUser` | Username for authentication (prompted via stdin if not configured) | + +## Package Store Types + +### basic (default) + +Downloads packages directly from `/.bolt` with no authentication. +The server can be a plain static file server — no special API is required. + +### rdk + +Downloads packages from an RDK package store server, authenticating with a username and password. +The password is prompted via stdin on first use and never stored. If `packageStoreUser` is not set +in the config, the username is also prompted. The session is preserved between invocations; when it +expires, credentials are prompted again automatically. + +Example configuration: +```json +{ + "packageStoreURL": "https://rdk.example.com", + "packageStoreType": "rdk", + "packageStoreUser": "myuser" +} +``` + +## Custom Package Store Types + +Custom types can be implemented as modules placed in `~/.bolt/plugins/`. The module file must be +named `fetch-.cjs` (e.g., `~/.bolt/plugins/fetch-custom.cjs` for type `"custom"`). + +The module exports a factory function that receives a context object and returns an async fetch +handler: + +```js +module.exports = function(ctx) { + return async function(packageStoreURL, packageFileName, options) { + await ctx.downloadPackage(`${packageStoreURL}/${packageFileName}`); + }; +}; +``` + +Available helpers on the `ctx` object: + +| Method | Description | +|---------------------------------------------|-----------------------------------------------------------| +| `ctx.downloadPackage(url, requestOptions?)` | Download a file with progress indicator | +| `ctx.postJSON(url, body)` | HTTP POST with JSON body, returns `{statusCode, headers, body}` | +| `ctx.promptCredentials(username?)` | Prompt for username/password via stdin | +| `ctx.loadData()` | Load saved data. Return `null` if no data | +| `ctx.saveData(data)` | Save data | + +## Examples + +- Download a package into the local package store: +``` +bolt fetch com.rdkcentral.base+0.2.0 +``` +- Same, using the file name form: +``` +bolt fetch com.rdkcentral.base+0.2.0.bolt +``` +- Download an architecture-specific package from a subdirectory on the server: +``` +bolt fetch arm/com.rdkcentral.base+0.2.0 +``` +- Re-download, replacing an existing copy: +``` +bolt fetch com.rdkcentral.base+0.2.0 --force +``` diff --git a/bolt/docs/local-package-store.md b/bolt/docs/local-package-store.md index 8cac1ec..3074d1e 100644 --- a/bolt/docs/local-package-store.md +++ b/bolt/docs/local-package-store.md @@ -5,6 +5,7 @@ It is used by: - `bolt make` — to resolve [package dependencies](make.md#dependency-handling) and to install built packages when `--install` or `--force-install` is used. +- `bolt fetch` — to [download packages](fetch.md) from a remote package store server. - `bolt push` — to [locate packages by name](push.md#locating-the-package) when no file path is provided. diff --git a/bolt/src/bolt.cjs b/bolt/src/bolt.cjs index 2ce8a22..ea278c0 100644 --- a/bolt/src/bolt.cjs +++ b/bolt/src/bolt.cjs @@ -27,6 +27,7 @@ const { pack, packOptions } = require('./pack.cjs'); const { push, pushOptions } = require('./push.cjs'); const { run, runOptions } = require('./run.cjs'); const { make, makeOptions } = require('./make.cjs'); +const { fetch, fetchOptions } = require('./fetch.cjs'); function help() { console.log(` @@ -53,6 +54,13 @@ Usage: Copy a bolt package to a remote device via SSH and optionally install it via middleware --direct Skip middleware installation and deploy directly to the bolt packages directory + bolt fetch [--force] + Download a bolt package from the configured package store server into the local package store + can be a package name (id+version) or file name (id+version.bolt) + --force Replace the package if it already exists in the local package store + Package store URL, type and credentials are configured in ~/.bolt/config.json + See https://github.com/rdkcentral/bolt-tools/blob/main/bolt/docs/fetch.md + bolt run [options] Execute a bolt package on a remote device. If a package ID is provided (without version), the package is always launched via middleware. @@ -102,6 +110,7 @@ const commands = { push: { args: 2, handler: push, options: pushOptions }, run: { args: 2, handler: run, options: runOptions }, make: { args: 1, handler: make, options: makeOptions }, + fetch: { args: 1, handler: fetch, options: fetchOptions }, }; function checkOptions(provided, allowed) { diff --git a/bolt/src/fetch-basic.cjs b/bolt/src/fetch-basic.cjs new file mode 100644 index 0000000..c96f108 --- /dev/null +++ b/bolt/src/fetch-basic.cjs @@ -0,0 +1,24 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2026 RDK Management + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +module.exports = function(ctx) { + return async function(packageStoreURL, packageFileName) { + await ctx.downloadPackage(`${packageStoreURL}/${packageFileName}`); + }; +}; diff --git a/bolt/src/fetch-rdk.cjs b/bolt/src/fetch-rdk.cjs new file mode 100644 index 0000000..023416c --- /dev/null +++ b/bolt/src/fetch-rdk.cjs @@ -0,0 +1,67 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2026 RDK Management + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +async function login(ctx, base, credentials) { + const res = await ctx.postJSON(`${base}/auth/login`, { + username: credentials.username, + password: credentials.password, + }); + + if (res.statusCode !== 200) { + throw new Error(`Authentication failed: HTTP ${res.statusCode}`); + } + + const setCookie = res.headers['set-cookie']; + if (!setCookie) { + throw new Error('Authentication failed: no cookie received from server'); + } + + const cookieHeader = Array.isArray(setCookie) ? setCookie[0] : setCookie; + return cookieHeader.split(';')[0].trim(); +} + +module.exports = function(ctx) { + return async function(packageStoreURL, packageFileName, options) { + const packagePath = packageFileName.includes('/') ? packageFileName : `arm/${packageFileName}`; + const url = `${packageStoreURL}/appcatalog/bolts/${packagePath}`; + let cookie = ctx.loadData(); + let prompted = false; + + if (!cookie) { + const credentials = await ctx.promptCredentials(options.packageStoreUser); + prompted = true; + cookie = await login(ctx, packageStoreURL, credentials); + ctx.saveData(cookie); + } + + try { + await ctx.downloadPackage(url, { headers: { Cookie: cookie } }); + } catch (err) { + if ((err.statusCode === 401 || err.statusCode === 403) && !prompted) { + console.log('Session expired, re-authenticating...'); + const credentials = await ctx.promptCredentials(options.packageStoreUser); + cookie = await login(ctx, packageStoreURL, credentials); + ctx.saveData(cookie); + await ctx.downloadPackage(url, { headers: { Cookie: cookie } }); + } else { + throw err; + } + } + }; +}; diff --git a/bolt/src/fetch.cjs b/bolt/src/fetch.cjs new file mode 100644 index 0000000..062fbae --- /dev/null +++ b/bolt/src/fetch.cjs @@ -0,0 +1,272 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2026 RDK Management + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +const http = require('node:http'); +const https = require('node:https'); +const fs = require('node:fs'); +const path = require('node:path'); +const readline = require('node:readline/promises'); +const { Transform } = require('node:stream'); +const { pipeline } = require('node:stream/promises'); +const { spawnSync } = require('node:child_process'); +const { Package } = require('./Package.cjs'); +const { PackageStore } = require('./PackageStore.cjs'); +const { GLOBAL_CONFIG_PATH } = require('./config.cjs'); +const fetchBasic = require('./fetch-basic.cjs'); +const fetchRdk = require('./fetch-rdk.cjs'); + +const CONFIG_DIR = path.dirname(GLOBAL_CONFIG_PATH); +const DATA_DIR = path.join(CONFIG_DIR, 'data'); +const PLUGINS_DIR = path.join(CONFIG_DIR, 'plugins'); + +class HTTPError extends Error { + constructor(statusCode, url) { + super(`HTTP ${statusCode}: ${url}`); + this.statusCode = statusCode; + } +} + +async function openURL(url, requestOptions = {}) { + for (let i = 0; i <= 5; i++) { + const protocol = url.startsWith('https://') ? https : http; + const res = await new Promise((resolve, reject) => { + protocol.get(url, requestOptions, resolve).on('error', reject); + }); + + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + res.resume(); + const redirectURL = new URL(res.headers.location, url).href; + if (!redirectURL.startsWith('http://') && !redirectURL.startsWith('https://')) { + throw new Error(`Refusing to follow redirect to non-HTTP URL: ${redirectURL}`); + } + if (url.startsWith('https://') && !redirectURL.startsWith('https://')) { + throw new Error(`Refusing to follow HTTPS to HTTP redirect for ${url}`); + } + url = redirectURL; + console.log(`Redirected to ${url}`); + continue; + } + + if (res.statusCode !== 200) { + res.resume(); + throw new HTTPError(res.statusCode, url); + } + + return res; + } + throw new Error(`Too many redirects for ${url}`); +} + +async function download(url, dest, requestOptions = {}) { + const tmpPath = dest + '~'; + try { + const res = await openURL(url, requestOptions); + const totalSize = parseInt(res.headers['content-length'], 10) || 0; + let downloaded = 0; + let lastPrint = 0; + + const printProgress = () => { + const mib = (downloaded / 1024 / 1024).toFixed(1); + const percent = totalSize ? ` (${Math.round(downloaded / totalSize * 100)}%)` : ''; + process.stdout.write(`\rDownloading... ${mib} MiB${percent}`); + }; + + const progress = new Transform({ + transform(chunk, encoding, callback) { + downloaded += chunk.length; + const now = Date.now(); + if (now - lastPrint >= 100) { + lastPrint = now; + printProgress(); + } + callback(null, chunk); + } + }); + + await pipeline(res, progress, fs.createWriteStream(tmpPath)); + printProgress(); + fs.renameSync(tmpPath, dest); + } catch (e) { + fs.rmSync(tmpPath, { force: true }); + throw e; + } finally { + process.stdout.write('\n'); + } +} + +function postJSON(url, body) { + return new Promise((resolve, reject) => { + const payload = JSON.stringify(body); + const protocol = url.startsWith('https://') ? https : http; + const req = protocol.request(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(payload), + }, + }, (res) => { + let data = ''; + res.setEncoding('utf8'); + res.on('data', chunk => data += chunk); + res.on('end', () => resolve({ statusCode: res.statusCode, headers: res.headers, body: data })); + }); + req.on('error', reject); + req.end(payload); + }); +} + +async function promptCredentials(username) { + if (!process.stdin.isTTY) { + throw new Error('Cannot prompt for credentials: stdin is not a terminal'); + } + const rl = readline.createInterface({ input: process.stdin, output: process.stderr, terminal: false }); + try { + username = username || await rl.question('Username: '); + const echoOff = spawnSync('stty', ['-echo'], { stdio: 'inherit' }).status === 0; + try { + const password = await rl.question('Password: '); + if (echoOff) process.stderr.write('\n'); + return { username, password }; + } finally { + if (echoOff) spawnSync('stty', ['echo'], { stdio: 'inherit' }); + } + } finally { + rl.close(); + } +} + +function loadServerData(dataPath) { + try { + return fs.readFileSync(dataPath, 'utf-8').trim(); + } catch (err) { + if (err.code === 'ENOENT') return null; + throw err; + } +} + +function saveServerData(dataPath, data) { + fs.mkdirSync(DATA_DIR, { recursive: true }); + fs.writeFileSync(dataPath, data, { mode: 0o600 }); +} + +function getServerDataPath(serverURL, type) { + const name = serverURL.replace(/[^a-zA-Z0-9.-]/g, '_'); + return path.join(DATA_DIR, `${name}.${type}.data`); +} + +function loadMethod(name) { + switch (name) { + case 'basic': return fetchBasic; + case 'rdk': return fetchRdk; + } + + const customPath = path.resolve(PLUGINS_DIR, `fetch-${name}.cjs`); + if (!customPath.startsWith(path.join(PLUGINS_DIR, ''))) { + throw new Error(`Invalid package store type: "${name}"`); + } + let stat; + try { + stat = fs.statSync(customPath); + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error(`Unknown package store type: "${name}". No fetch-${name}.cjs found in ${PLUGINS_DIR}`); + } + throw err; + } + + if (!stat.isFile()) { + throw new Error(`Refusing to load ${customPath}: not a regular file`); + } + + if (process.getuid) { + if (stat.uid !== process.getuid()) { + throw new Error(`Refusing to load ${customPath}: not owned by the current user`); + } + if (stat.mode & 0o022) { + throw new Error(`Refusing to load ${customPath}: writable by group or others (mode ${(stat.mode & 0o777).toString(8)})`); + } + } + + const method = require(customPath); + if (typeof method !== 'function') { + throw new Error(`Plugin ${customPath} does not export a function`); + } + return method; +} + +async function fetch(packageName, options) { + if (!options.packageStoreURL) { + throw new Error('No package store URL configured. Set "packageStoreURL" in ~/.bolt/config.json'); + } + + const packageStore = PackageStore.find(process.cwd()); + if (!packageStore) { + throw new Error('Local package store not found'); + } + + const packageFileName = Package.isPackageFileName(packageName) + ? packageName + : Package.makeFileName(packageName); + const packageFullName = Package.pathToFullName(packageFileName); + const dest = packageStore.generatePackagePath(packageFullName); + + const destStat = fs.statSync(dest, { throwIfNoEntry: false }); + if (destStat && !destStat.isFile()) { + throw new Error(`${packageFileName} (${dest}) exists but is not a regular file`); + } + if (destStat && !options.force) { + throw new Error(`${packageFileName} already exists in the local package store. Use --force to replace it.`); + } + + const base = options.packageStoreURL.endsWith('/') ? options.packageStoreURL.slice(0, -1) : options.packageStoreURL; + const methodName = options.packageStoreType || 'basic'; + const dataPath = getServerDataPath(options.packageStoreURL, methodName); + const method = loadMethod(methodName); + const handler = method({ + postJSON, + promptCredentials, + downloadPackage: (url, opts) => download(url, dest, opts), + loadData: () => loadServerData(dataPath), + saveData: (data) => saveServerData(dataPath, data), + }); + + await handler(base, packageFileName, options); + + console.log(`Fetched ${packageFileName} into ${packageStore.getPath()}`); +} + +exports.fetch = fetch; + +exports.fetchOptions = { + packageStoreURL(params, result) { + return !!(result.packageStoreURL = params.options.packageStoreURL); + }, + + packageStoreType(params, result) { + return !!(result.packageStoreType = params.options.packageStoreType); + }, + + packageStoreUser(params, result) { + return !!(result.packageStoreUser = params.options.packageStoreUser); + }, + + force(params, result) { + return (result.force = (params.options.force === '')); + }, +}; diff --git a/bolt/src/globalConfig.cjs b/bolt/src/globalConfig.cjs index 1e9ea5b..b1748a5 100644 --- a/bolt/src/globalConfig.cjs +++ b/bolt/src/globalConfig.cjs @@ -24,6 +24,15 @@ const path = require('node:path'); const CONFIG_DIR = path.dirname(GLOBAL_CONFIG_PATH); +function isHTTPURL(value) { + try { + const { protocol } = new URL(value); + return protocol === 'http:' || protocol === 'https:'; + } catch { + return false; + } +} + function loadGlobalConfig() { let parsed; try { @@ -53,6 +62,26 @@ function loadGlobalConfig() { console.warn(`Warning: invalid 'cert' in ${GLOBAL_CONFIG_PATH}: expected a non-empty string`); } + if (parsed?.packageStoreURL !== undefined) { + if (isHTTPURL(parsed.packageStoreURL)) { + result.packageStoreURL = parsed.packageStoreURL; + } else { + console.warn(`Warning: invalid 'packageStoreURL' in ${GLOBAL_CONFIG_PATH}: expected an HTTP or HTTPS URL`); + } + } + + if (typeof parsed?.packageStoreType === 'string' && parsed.packageStoreType !== '') { + result.packageStoreType = parsed.packageStoreType; + } else if (parsed?.packageStoreType !== undefined) { + console.warn(`Warning: invalid 'packageStoreType' in ${GLOBAL_CONFIG_PATH}: expected a non-empty string`); + } + + if (typeof parsed?.packageStoreUser === 'string' && parsed.packageStoreUser !== '') { + result.packageStoreUser = parsed.packageStoreUser; + } else if (parsed?.packageStoreUser !== undefined) { + console.warn(`Warning: invalid 'packageStoreUser' in ${GLOBAL_CONFIG_PATH}: expected a non-empty string`); + } + return result; }