feat: modules as plugins #1002

companion/lib/Data/ImportExport.ts

@@ -887,7 +887,12 @@ export class DataImportExport extends CoreBase {
 				} else {
 					// Create a new instance
 					const instance_type = this.instance.modules.verifyInstanceTypeIsCurrent(obj.instance_type)
-					const [newId, newConfig] = this.instance.addInstanceWithLabel({ type: instance_type }, obj.label, true)
+					const [newId, newConfig] = this.instance.addInstanceWithLabel(
+						{ type: instance_type },
+						obj.label,
+						{ mode: 'stable', id: null }, // Always create using the latest stable
+						true
+					)
 					console.log('created', instance_type, newId)
 					if (newId && newConfig) {
 						this.instance.setInstanceLabelAndConfig(newId, null, 'config' in obj ? obj.config : null)

companion/lib/Data/StoreBase.ts

@@ -192,8 +192,8 @@ export abstract class DataStoreBase {
 	 * @param table - the table to get from
 	 * @returns the rows
 	 */
-	public getTable(table: string): any {
-		let out = {}
+	public getTable(table: string): Record<string, any> {
+		let out: Record<string, any> = {}
 
 		if (table.length > 0) {
 			const query = this.store.prepare(`SELECT id, value FROM ${table}`)

companion/lib/Instance/ConnectionConfigStore.ts

@@ -5,6 +5,7 @@ import { nanoid } from 'nanoid'
 import { cloneDeep } from 'lodash-es'
 import { ClientConnectionConfig } from '@companion-app/shared/Model/Common.js'
 import { makeLabelSafe } from '@companion-app/shared/Label.js'
+import { ModuleVersionInfo } from '@companion-app/shared/Model/ModuleInfo.js'
 
 export class ConnectionConfigStore {
 	// readonly #logger = LogController.createLogger('Instance/ConnectionConfigStore')
@@ -55,6 +56,7 @@ export class ConnectionConfigStore {
 		moduleType: string,
 		label: string,
 		product: string | undefined,
+		moduleVersion: ModuleVersionInfo,
 		disabled: boolean
 	): [id: string, config: ConnectionConfig] {
 		// Find the highest rank given to an instance
@@ -70,6 +72,8 @@ export class ConnectionConfigStore {
 
 		this.#store[id] = {
 			instance_type: moduleType,
+			moduleVersionMode: moduleVersion.mode,
+			moduleVersionId: moduleVersion?.id || null,
 			sortOrder: highestRank + 1,
 			label: label,
 			isFirstInit: true,
@@ -102,6 +106,8 @@ export class ConnectionConfigStore {
 
 			result[id] = {
 				instance_type: config.instance_type,
+				moduleVersionMode: config.moduleVersionMode,
+				moduleVersionId: config.moduleVersionId,
 				label: config.label,
 				enabled: config.enabled,
 				sortOrder: config.sortOrder,

companion/lib/Instance/Controller.ts

@@ -32,6 +32,13 @@ import type { ModuleManifest } from '@companion-module/base'
 import type { ExportInstanceFullv4, ExportInstanceMinimalv4 } from '@companion-app/shared/Model/ExportModel.js'
 import type { ClientSocket } from '../UI/Handler.js'
 import { ConnectionConfigStore } from './ConnectionConfigStore.js'
+import { InstanceInstalledModulesManager } from './InstalledModulesManager.js'
+import type { ModuleVersionInfo } from '@companion-app/shared/Model/ModuleInfo.js'
+import type { ModuleDirs } from './Types.js'
+import path from 'path'
+import { isPackaged } from '../Resources/Util.js'
+import { fileURLToPath } from 'url'
+import { ModuleStoreService } from './ModuleStore.js'
 
 const InstancesRoom = 'instances'
 
@@ -58,6 +65,8 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 	readonly status: InstanceStatus
 	readonly moduleHost: ModuleHost
 	readonly modules: InstanceModules
+	readonly modulesStore: ModuleStoreService
+	readonly userModulesManager: InstanceInstalledModulesManager
 
 	constructor(registry: Registry) {
 		super(registry, 'Instance/Controller')
@@ -67,10 +76,31 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 
 		this.#configStore = new ConnectionConfigStore(registry.db, this.broadcastChanges.bind(this))
 
+		function generatePath(subpath: string): string {
+			if (isPackaged()) {
+				return path.join(__dirname, subpath)
+			} else {
+				return fileURLToPath(new URL(path.join('../../..', subpath), import.meta.url))
+			}
+		}
+
+		const moduleDirs: ModuleDirs = {
+			bundledLegacyModulesDir: path.resolve(generatePath('modules')),
+			bundledModulesDir: path.resolve(generatePath('bundled-modules')),
+			installedModulesDir: path.join(registry.appInfo.modulesDir, 'store'),
+		}
+
 		this.definitions = new InstanceDefinitions(registry)
 		this.status = new InstanceStatus(registry.io, registry.controls)
 		this.moduleHost = new ModuleHost(registry, this.status, this.#configStore)
-		this.modules = new InstanceModules(registry)
+		this.modules = new InstanceModules(registry.io, registry.api_router, this, moduleDirs)
+		this.modulesStore = new ModuleStoreService(registry.io, registry.data.cache)
+		this.userModulesManager = new InstanceInstalledModulesManager(
+			registry.appInfo,
+			this.modules,
+			this.modulesStore,
+			moduleDirs
+		)
 
 		// Prepare for clients already
 		this.broadcastChanges(this.#configStore.getAllInstanceIds())
@@ -104,19 +134,22 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 	 * @param extraModulePath - extra directory to search for modules
 	 */
 	async initInstances(extraModulePath: string): Promise<void> {
-		const connectionIds = this.#configStore.getAllInstanceIds()
-		this.logger.silly('instance_init', connectionIds)
+		await this.userModulesManager.init()
 
 		await this.modules.initInstances(extraModulePath)
 
+		const connectionIds = this.#configStore.getAllInstanceIds()
+		this.logger.silly('instance_init', connectionIds)
 		for (const id of connectionIds) {
 			this.#activate_module(id, false)
 		}
 
 		this.emit('connection_added')
 	}
 
-	reloadUsesOfModule(moduleId: string): void {
+	async reloadUsesOfModule(moduleId: string, mode: 'release' | 'dev', versionId: string | null): Promise<void> {
+		// TODO - use the version!
+
 		// restart usages of this module
 		const { connectionIds, labels } = this.#configStore.findActiveUsagesOfModule(moduleId)
 		for (const id of connectionIds) {
@@ -180,36 +213,28 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 		this.logger.debug(`instance "${connectionConfig.label}" configuration updated`)
 	}
 
-	/**
-	 * Add a new instance of a module
-	 */
-	addInstance(data: CreateConnectionData, disabled: boolean): string {
-		let module = data.type
-
-		const moduleInfo = this.modules.getModuleManifest(module)
-		if (!moduleInfo) throw new Error(`Unknown module type ${module}`)
-
-		return this.addInstanceWithLabel(data, moduleInfo.display.shortname, disabled)[0]
-	}
-
 	/**
 	 * Add a new instance of a module with a predetermined label
 	 */
 	addInstanceWithLabel(
 		data: CreateConnectionData,
 		labelBase: string,
+		version: ModuleVersionInfo,
 		disabled: boolean
 	): [id: string, config: ConnectionConfig] {
 		let module = data.type
 		let product = data.product
 
+		const moduleInfo = this.modules.getModuleManifest(module, version.mode, version.id)
+		if (!moduleInfo) throw new Error(`Unknown module type ${module}`)
+
 		const label = this.#configStore.makeLabelUnique(labelBase)
 
 		if (this.getIdForLabel(label)) throw new Error(`Label "${label}" already in use`)
 
 		this.logger.info('Adding connection ' + module + ' ' + product)
 
-		const [id, config] = this.#configStore.addConnection(module, label, product, disabled)
+		const [id, config] = this.#configStore.addConnection(module, label, product, version, disabled)
 
 		this.#activate_module(id, true)
 
@@ -233,7 +258,11 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 		const config = this.#configStore.getConfigForId(id)
 		if (!config) return undefined
 
-		const moduleManifest = this.modules.getModuleManifest(config.instance_type)
+		const moduleManifest = this.modules.getModuleManifest(
+			config.instance_type,
+			config.moduleVersionMode,
+			config.moduleVersionId
+		)
 
 		return moduleManifest?.manifest
 	}
@@ -394,6 +423,12 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 
 		config.instance_type = this.modules.verifyInstanceTypeIsCurrent(config.instance_type)
 
+		// Seamless fixup old configs
+		if (config.moduleVersionMode === undefined) {
+			config.moduleVersionMode = 'stable'
+			config.moduleVersionId = null
+		}
+
 		if (config.enabled === false) {
 			this.logger.silly("Won't load disabled module " + id + ' (' + config.instance_type + ')')
 			this.status.updateInstanceStatus(id, null, 'Disabled')
@@ -412,9 +447,20 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 		// TODO this could check if anything above changed, or is_being_created
 		this.#configStore.commitChanges([id])
 
-		const moduleInfo = this.modules.getModuleManifest(config.instance_type)
+		const moduleInfo = this.modules.getModuleManifest(
+			config.instance_type,
+			config.moduleVersionMode,
+			config.moduleVersionId
+		)
 		if (!moduleInfo) {
 			this.logger.error('Configured instance ' + config.instance_type + ' could not be loaded, unknown module')
+			if (this.modules.hasModule(config.instance_type)) {
+				// TODO - check level
+				this.status.updateInstanceStatus(id, null, 'Unknown module version')
+			} else {
+				// TODO - check level
+				this.status.updateInstanceStatus(id, null, 'Unknown module')
+			}
 		} else {
 			this.moduleHost.queueRestartConnection(id, config, moduleInfo).catch((e) => {
 				this.logger.error('Configured instance ' + config.instance_type + ' failed to start: ', e)
@@ -430,6 +476,8 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 		this.definitions.clientConnect(client)
 		this.status.clientConnect(client)
 		this.modules.clientConnect(client)
+		this.modulesStore.clientConnect(client)
+		this.userModulesManager.clientConnect(client)
 
 		client.onPromise('connections:subscribe', () => {
 			client.join(InstancesRoom)
@@ -462,7 +510,7 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 			}
 		})
 
-		client.onPromise('connections:set-config', (id, label, config) => {
+		client.onPromise('connections:set-label-and-config', (id, label, config) => {
 			const idUsingLabel = this.getIdForLabel(label)
 			if (idUsingLabel && idUsingLabel !== id) {
 				return 'duplicate label'
@@ -477,6 +525,43 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 			return null
 		})
 
+		client.onPromise('connections:set-label-and-version', (id, label, version) => {
+			const idUsingLabel = this.getIdForLabel(label)
+			if (idUsingLabel && idUsingLabel !== id) {
+				return 'duplicate label'
+			}
+
+			if (!isLabelValid(label)) {
+				return 'invalid label'
+			}
+
+			// TODO - refactor/optimise/tidy this
+
+			this.setInstanceLabelAndConfig(id, label, null)
+
+			const config = this.#configStore.getConfigForId(id)
+			if (!config) return 'no connection'
+
+			const moduleInfo = this.modules.getModuleManifest(config.instance_type, version.mode, version.id)
+			if (!moduleInfo)
+				throw new Error(`Unknown module type or version ${config.instance_type} (${version.mode}:${version.id})`)
+
+			// Update the config
+			config.moduleVersionMode = version.mode
+			config.moduleVersionId = version.id
+			this.#configStore.commitChanges([id])
+
+			console.log('new config', config)
+
+			// Trigger a restart
+			if (config.enabled) {
+				this.enableDisableInstance(id, false)
+				this.enableDisableInstance(id, true)
+			}
+
+			return null
+		})
+
 		client.onPromise('connections:set-enabled', (id, state) => {
 			this.enableDisableInstance(id, !!state)
 		})
@@ -485,9 +570,9 @@ export class InstanceController extends CoreBase<InstanceControllerEvents> {
 			await this.deleteInstance(id)
 		})
 
-		client.onPromise('connections:add', (module) => {
-			const id = this.addInstance(module, false)
-			return id
+		client.onPromise('connections:add', (module, label, version) => {
+			const connectionInfo = this.addInstanceWithLabel(module, label, version, false)
+			return connectionInfo[0]
 		})
 
 		client.onPromise('connections:set-order', async (connectionIds) => {

companion/lib/Instance/Host.ts

@@ -11,9 +11,9 @@ import { RespawnMonitor } from '@companion-app/shared/Respawn.js'
 import type { Registry } from '../Registry.js'
 import type { InstanceStatus } from './Status.js'
 import type { ConnectionConfig } from '@companion-app/shared/Model/Connections.js'
-import type { ModuleInfo } from './Modules.js'
 import type { ConnectionConfigStore } from './ConnectionConfigStore.js'
 import { isModuleApiVersionCompatible } from '@companion-app/shared/ModuleApiVersionCheck.js'
+import type { SomeModuleVersionInfo } from './Types.js'
 
 /**
  * A backoff sleep strategy
@@ -93,7 +93,13 @@ export class ModuleHost {
 			if (!child.crashed) {
 				child.crashed = setTimeout(() => {
 					const config = this.#connectionConfigStore.getConfigForId(child.connectionId)
-					const moduleInfo = config && this.#registry.instance.modules.getModuleManifest(config.instance_type)
+					const moduleInfo =
+						config &&
+						this.#registry.instance.modules.getModuleManifest(
+							config.instance_type,
+							config.moduleVersionMode,
+							config.moduleVersionId
+						)
 
 					// Restart after a short sleep
 					this.queueRestartConnection(child.connectionId, config, moduleInfo)
@@ -331,7 +337,7 @@ export class ModuleHost {
 	async queueRestartConnection(
 		connectionId: string,
 		config: ConnectionConfig | undefined,
-		moduleInfo: ModuleInfo | undefined
+		moduleInfo: SomeModuleVersionInfo | undefined
 	): Promise<void> {
 		if (!config || !moduleInfo) return