diff --git a/apps/cli/package.json b/apps/cli/package.json index a95760a0..e603aa76 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -27,6 +27,7 @@ "@usevenice/connector-toggl": "workspace:*", "@usevenice/connector-venmo": "workspace:*", "@usevenice/connector-wise": "workspace:*", + "@usevenice/connector-xero": "workspace:*", "@usevenice/connector-yodlee": "workspace:*", "@usevenice/engine-backend": "workspace:*", "@usevenice/env": "workspace:*", diff --git a/connectors/connector-xero/def.ts b/connectors/connector-xero/def.ts index 7f6624ab..a2b4d12c 100644 --- a/connectors/connector-xero/def.ts +++ b/connectors/connector-xero/def.ts @@ -1,21 +1,32 @@ -import type { - ConnectorDef, - ConnectorSchemas, - EntityPayloadWithRaw, -} from '@usevenice/cdk' -import {connHelpers, oauthBaseSchema} from '@usevenice/cdk' -import {z, zCast} from '@usevenice/util' +import type {components} from '@opensdks/sdk-xero/xero_accounting.oas.types' +import type {ConnectorDef, ConnectorSchemas} from '@usevenice/cdk' +import {connHelpers, oauthBaseSchema, zEntityPayload} from '@usevenice/cdk' +import {R, z} from '@usevenice/util' + +export const zConfig = oauthBaseSchema.connectorConfig + +const oReso = oauthBaseSchema.resourceSettings +export const zSettings = oReso.extend({ + oauth: oReso.shape.oauth, +}) + +export type XERO = components['schemas'] + +export const XERO_ENTITY_NAME = { + Account: 'Account', + BankTransaction: 'BankTransaction', +} as const export const xeroSchemas = { name: z.literal('xero'), - resourceSettings: z.object({ - access_token: z.string(), - }), - destinationInputEntity: zCast(), - connectorConfig: oauthBaseSchema.connectorConfig, + connectorConfig: zConfig, + resourceSettings: zSettings, + connectOutput: oauthBaseSchema.connectOutput, + sourceOutputEntity: zEntityPayload, + sourceOutputEntities: R.mapValues(XERO_ENTITY_NAME, () => z.unknown()), } satisfies ConnectorSchemas -export const helpers = connHelpers(xeroSchemas) +export const xeroHelpers = connHelpers(xeroSchemas) export const xeroDef = { metadata: { diff --git a/connectors/connector-xero/index.ts b/connectors/connector-xero/index.ts index 81f31bd7..d180f7cb 100644 --- a/connectors/connector-xero/index.ts +++ b/connectors/connector-xero/index.ts @@ -1,4 +1,10 @@ +import type {initXeroSDK} from '@opensdks/sdk-xero' + // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} export * from './def' export * from './server' // codegen:end + +export * from '@opensdks/sdk-xero' + +export type XeroSDK = ReturnType diff --git a/connectors/connector-xero/package.json b/connectors/connector-xero/package.json index 9e1e7663..dd9423d3 100644 --- a/connectors/connector-xero/package.json +++ b/connectors/connector-xero/package.json @@ -5,7 +5,8 @@ "sideEffects": [], "module": "./index.ts", "dependencies": { - "@opensdks/sdk-xero": "^0.0.2", + "@opensdks/runtime": "^0.0.19", + "@opensdks/sdk-xero": "^0.0.7", "@opensdks/util-zod": "^0.0.15", "@usevenice/cdk": "workspace:*", "@usevenice/util": "workspace:*" diff --git a/connectors/connector-xero/server.ts b/connectors/connector-xero/server.ts index 88732094..7d0e497e 100644 --- a/connectors/connector-xero/server.ts +++ b/connectors/connector-xero/server.ts @@ -1,16 +1,83 @@ import {initXeroSDK} from '@opensdks/sdk-xero' import type {ConnectorServer} from '@usevenice/cdk' -import type {xeroSchemas} from './def' +import {nangoProxyLink} from '@usevenice/cdk' +import {Rx, rxjs} from '@usevenice/util' +import {XERO_ENTITY_NAME, xeroHelpers, type xeroSchemas} from './def' export const xeroServer = { - newInstance: ({settings}) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + // Would be good if this was async... + newInstance: ({settings, fetchLinks}) => { const xero = initXeroSDK({ - headers: {authorization: `Bearer ${settings.access_token}`}, + headers: { + authorization: `Bearer ${settings.oauth.credentials.access_token}`, + }, + links: (defaultLinks) => [ + (req, next) => { + req.headers.set( + nangoProxyLink.kBaseUrlOverride, + 'https://api.xero.com', + ) + // nango's proxy endpoint is pretty annoying... Will only proxy + // if it is prefixed with nango-proxy. Might as well not proxy like this... + const tenantId = req.headers.get('xero-tenant-id') + if (tenantId) { + req.headers.delete('xero-tenant-id') + req.headers.set('nango-proxy-xero-tenant-id', tenantId) + } + return next(req) + }, + ...fetchLinks, + ...defaultLinks, + ], }) - // TODO(@jatin): Add logic here to handle sync. return xero }, -} satisfies ConnectorServer + sourceSync: ({instance: xero, streams}) => { + console.log('[xero] Starting sync') + async function* iterateEntities() { + // TODO: Should handle more than one tenant Id + const tenantId = await xero.identity + .GET('/Connections') + .then((r) => r.data?.[0]?.tenantId) + if (!tenantId) { + throw new Error( + 'Missing access to any tenants. Check xero token permission', + ) + } + for (const type of Object.values(XERO_ENTITY_NAME)) { + if (!streams[type]) { + continue + } + + const singular = type as 'BankTransaction' + const plural = `${singular}s` as const + const kId = `${singular}ID` as const + let page = 1 + while (true) { + const result = await xero.accounting.GET(`/${plural}`, { + params: {header: {'xero-tenant-id': tenantId}, query: {page}}, + }) + if (result.data[plural]?.length) { + yield result.data[plural]?.map((a) => + xeroHelpers._opData(singular, a[kId]!, a), + ) + // Account does not support pagination, all or nothing... + if (type !== 'Account') { + page++ + continue + } + } + break + } + } + } + + return rxjs + .from(iterateEntities()) + .pipe( + Rx.mergeMap((ops) => rxjs.from([...ops, xeroHelpers._op('commit')])), + ) + }, +} satisfies ConnectorServer> export default xeroServer diff --git a/docs/samples/banking-test.ts b/docs/samples/banking-test.ts index cbc6ef83..51bdb90d 100644 --- a/docs/samples/banking-test.ts +++ b/docs/samples/banking-test.ts @@ -3,7 +3,8 @@ import {createVeniceClient} from '@usevenice/sdk' const venice = createVeniceClient({ apiKey: process.env['_VENICE_API_KEY'], apiHost: process.env['_VENICE_API_HOST'], - resourceId: process.env['_QBO_RESOURCE_ID'], + // resourceId: process.env['_QBO_RESOURCE_ID'], + resourceId: process.env['_XERO_RESOURCE_ID'], }) void venice.GET('/verticals/banking/category').then((r) => { diff --git a/packages/engine-backend/router/_base.ts b/packages/engine-backend/router/_base.ts index f1d7d9db..38f5e8ac 100644 --- a/packages/engine-backend/router/_base.ts +++ b/packages/engine-backend/router/_base.ts @@ -45,19 +45,18 @@ export const trpc = initTRPC export const publicProcedure = trpc.procedure // Enable me for logs -// .use( -// ({next, ctx, input, rawInput, meta, path}) => { -// console.log('[trpc]', { -// input, -// rawInput, -// meta, -// path, -// }) -// return next({ctx}) -// }, -// ) +// .use(({next, ctx, input, rawInput, meta, path}) => { +// console.log('[trpc]', { +// input, +// rawInput, +// meta, +// path, +// }) +// return next({ctx}) +// }) export const protectedProcedure = publicProcedure.use(({next, ctx}) => { + console.log('DEBUG', ctx.viewer) if (!hasRole(ctx.viewer, ['end_user', 'user', 'org', 'system'])) { throw new TRPCError({ code: ctx.viewer.role === 'anon' ? 'UNAUTHORIZED' : 'FORBIDDEN', diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index 93492277..6b98e414 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -205,6 +205,8 @@ export const endUserRouter = trpc.router({ const int = await ctx.asOrgIfNeeded.getConnectorConfigOrFail(ccfgId) console.log('didConnect start', int.connector.name, input, connCtxInput) + // TODO: we should make it possible for oauth connectors to + // ALSO handle custom postConnect... This would be very handy for xero for instance const resoUpdate = await (async () => { if ( !int.connector.postConnect && diff --git a/packages/engine-backend/router/index.ts b/packages/engine-backend/router/index.ts index f0fec05b..01a46581 100644 --- a/packages/engine-backend/router/index.ts +++ b/packages/engine-backend/router/index.ts @@ -11,6 +11,7 @@ import { outreachAdapter, qboAdapter, salesloftAdapter, + xeroAdapter, } from '@usevenice/cdk/verticals' import {remoteProcedure, trpc} from './_base' import {adminRouter} from './adminRouter' @@ -26,7 +27,7 @@ import {systemRouter} from './systemRouter' const bankingRouter = createBankingRouter({ trpc, remoteProcedure, - adapterByName: {qbo: qboAdapter}, + adapterByName: {qbo: qboAdapter, xero: xeroAdapter}, }) const accountingRouter = createAccountingRouter({ trpc, diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index 9a90dc25..d831ccf8 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -156,7 +156,6 @@ export function _VeniceConnect({ }) { const nangoPublicKey = _trpcReact.getPublicEnv.useQuery().data?.NEXT_PUBLIC_NANGO_PUBLIC_KEY - const nangoFrontend = React.useMemo( () => nangoPublicKey && diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2ff9fd4..aed687c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -409,6 +409,9 @@ importers: '@usevenice/connector-wise': specifier: workspace:* version: link:../../connectors/connector-wise + '@usevenice/connector-xero': + specifier: workspace:* + version: link:../../connectors/connector-xero '@usevenice/connector-yodlee': specifier: workspace:* version: link:../../connectors/connector-yodlee @@ -492,6 +495,22 @@ importers: specifier: 2.5.0 version: 2.5.0 + apps/tests: + dependencies: + '@usevenice/app-config': + specifier: workspace:* + version: link:../app-config + remeda: + specifier: 1.6.1 + version: 1.6.1 + devDependencies: + openapi-typescript: + specifier: 6.7.1 + version: 6.7.1 + tsx: + specifier: 3.12.2 + version: 3.12.2 + apps/web: dependencies: '@clerk/nextjs': @@ -1165,9 +1184,12 @@ importers: connectors/connector-xero: dependencies: + '@opensdks/runtime': + specifier: ^0.0.19 + version: 0.0.19 '@opensdks/sdk-xero': - specifier: ^0.0.2 - version: 0.0.2 + specifier: ^0.0.7 + version: 0.0.7 '@opensdks/util-zod': specifier: ^0.0.15 version: 0.0.15 @@ -1808,6 +1830,9 @@ importers: '@usevenice/connector-qbo': specifier: workspace:* version: link:../../connectors/connector-qbo + '@usevenice/connector-xero': + specifier: workspace:* + version: link:../../connectors/connector-xero '@usevenice/types': specifier: workspace:* version: link:../../utils/types @@ -4709,6 +4734,10 @@ packages: /@opensdks/fetch-links@0.0.17: resolution: {integrity: sha512-BUomrnpFtlu085W1wOY0k52Kf6qkJwBDifQxFoSHkfjyjS81AkpOTPsBTYS5HffAXS8rgoBBJF+7fu2NSYipFw==} + /@opensdks/fetch-links@0.0.18: + resolution: {integrity: sha512-EhBj92wLXoj3pF4ERxzoEhAlwlefIzQ31At0zaksSbA9zqMlod9DyfypQRX/6J8w4iZzraV33BtFEk7l98nsLA==} + dev: false + /@opensdks/runtime@0.0.16: resolution: {integrity: sha512-aCRhxlWr2e+hB38quP2tLkaPGZNPMV9rBE+IoTwDoHS4mql2dyn7HvxRYisoU3XmdaMCLvbf8+H/7Nrj02tMpg==} dependencies: @@ -4717,6 +4746,15 @@ packages: openapi-typescript-helpers: 0.0.4 openapi3-ts: 4.1.2 + /@opensdks/runtime@0.0.19: + resolution: {integrity: sha512-9rPrPDHS+PRVKOWXbdYEtys/4uHeVzitQgR0QVSWtMFOn6FOY8VxMGvem0TGJwHf4EPecejoWF8NuNwUATmbeA==} + dependencies: + '@opensdks/fetch-links': 0.0.18 + openapi-fetch: 0.8.1 + openapi-typescript-helpers: 0.0.4 + openapi3-ts: 4.1.2 + dev: false + /@opensdks/sdk-apollo@0.0.18: resolution: {integrity: sha512-i2rps56eTt22cM1pwbVyTVlZtpjP65T3jMJhX2rKO2CiHd7+QoI3+hT+ko6dCIsi6STPnoM+IIJhmyQZjJIktA==} dev: false @@ -4764,8 +4802,10 @@ packages: '@opensdks/runtime': 0.0.16 dev: true - /@opensdks/sdk-xero@0.0.2: - resolution: {integrity: sha512-ni6s4q658K9dWsmXwD4mnJrL9QguZwGxgtDt24Is4cjUNmiLqfxpC7PqE4ecxKn/TNv+oZiQJKm7Di29UHt3rQ==} + /@opensdks/sdk-xero@0.0.7: + resolution: {integrity: sha512-NZLi+HH0Jrss+sCnLyDhnih24wjcz/u77zOvFtpBwbbwCUqPSYQ/s50qO0ek+6g9eJG3YxoeplOM51YbyszPwA==} + dependencies: + '@opensdks/runtime': 0.0.19 dev: false /@opensdks/util-zod@0.0.15: diff --git a/verticals/vertical-banking/adapters/xero-adapter.ts b/verticals/vertical-banking/adapters/xero-adapter.ts new file mode 100644 index 00000000..4bcc17f0 --- /dev/null +++ b/verticals/vertical-banking/adapters/xero-adapter.ts @@ -0,0 +1,46 @@ +import type {Oas_accounting, XeroSDK} from 'connectors/connector-xero' +import type {StrictObj} from '@usevenice/vdk' +import {mapper, z, zCast} from '@usevenice/vdk' +import type {VerticalBanking} from '../banking' +import {zBanking} from '../banking' + +type Xero = Oas_accounting['components']['schemas'] + +const mappers = { + category: mapper( + zCast>(), + zBanking.category.extend({_raw: z.unknown().optional()}), + { + id: 'AccountID', + name: 'Name', + _raw: (a) => a, + }, + ), +} + +export const xeroAdapter = { + listCategories: async ({instance}) => { + // TODO: Abstract this away please... + const tenantId = await instance.identity + .GET('/Connections') + .then((r) => r.data?.[0]?.tenantId) + if (!tenantId) { + throw new Error( + 'Missing access to any tenants. Check xero token permission', + ) + } + + const res = await instance.accounting.GET('/Accounts', { + params: { + header: {'xero-tenant-id': tenantId}, + query: { + where: 'Class=="REVENUE"||Class=="EXPENSE"', + }, + }, + }) + return { + hasNextPage: false, + items: (res.data.Accounts ?? []).map(mappers.category), + } + }, +} satisfies VerticalBanking<{instance: XeroSDK}> diff --git a/verticals/vertical-banking/banking.ts b/verticals/vertical-banking/banking.ts index 01efb6e2..bc8d4c39 100644 --- a/verticals/vertical-banking/banking.ts +++ b/verticals/vertical-banking/banking.ts @@ -4,6 +4,7 @@ import type {AnyEntityPayload, Id, Link} from '@usevenice/cdk' import type {PlaidSDKTypes} from '@usevenice/connector-plaid' import type {postgresHelpers} from '@usevenice/connector-postgres' import type {QBO} from '@usevenice/connector-qbo' +import type {Oas_accounting} from '@usevenice/connector-xero' import type {StrictObj} from '@usevenice/types' import type {RouterMap, RouterMeta, VerticalRouterOpts} from '@usevenice/vdk' import { @@ -16,6 +17,7 @@ import { } from '@usevenice/vdk' type Plaid = PlaidSDKTypes['oas']['components'] +type Xero = Oas_accounting['components']['schemas'] export const zBanking = { transaction: z @@ -83,6 +85,75 @@ export function bankingLink(ctx: { if (op.type !== 'data') { return rxjs.of(op) } + + if (ctx.source.connectorConfig.connectorName === 'xero') { + if (op.data.entityName === 'Account') { + const entity = op.data.entity as Xero['Account'] + if (entity.Class === 'REVENUE' || entity.Class === 'EXPENSE') { + const mapped = applyMapper( + mappers.xero.category, + op.data.entity as Xero['Account'], + ) + return rxjs.of({ + ...op, + data: { + id: mapped.id, + entityName: 'banking_category', + entity: {raw: op.data.entity, unified: mapped}, + } satisfies PostgresInputPayload, + }) + } else { + const mapped = applyMapper( + mappers.xero.account, + op.data.entity as Xero['Account'], + ) + return rxjs.of({ + ...op, + data: { + id: mapped.id, + entityName: 'banking_account', + entity: {raw: op.data.entity, unified: mapped}, + } satisfies PostgresInputPayload, + }) + } + } + if (op.data.entityName === 'BankTransaction') { + // TODO: Dedupe from qbo.purchase later + const mapped = applyMapper( + mappers.xero.bank_transaction, + op.data.entity as Xero['BankTransaction'], + ) + // TODO: Make this better, should at the minimum apply to both Plaid & QBO, options are + // 1) Banking link needs to take input parameters to determine if by default + // transactions should go through if metadata is missing or not + // 2) Banking vertical should include abstraction for account / category selection UI etc. + // 3) Extract this into a more generic filtering link that works for ANY entity. + // In addition, will need to handle incremental sync state reset when we change stream filtering + // parameter like this, as well as deleting the no longer relevant entities in destination + if ( + // Support both name and ID + !categories[mapped.category_name ?? ''] && + !categories[mapped.category_id ?? ''] + ) { + console.log( + `[banking] skip txn ${mapped.id} in ${mapped.category_id}: ${mapped.category_name}`, + ) + return rxjs.EMPTY + } else { + console.log( + `[banking] allow txn ${mapped.id} in ${mapped.category_id}: ${mapped.category_name}`, + ) + } + return rxjs.of({ + ...op, + data: { + id: mapped.id, + entityName: 'banking_transaction', + entity: {raw: op.data.entity, unified: mapped}, + } satisfies PostgresInputPayload, + }) + } + } if (ctx.source.connectorConfig.connectorName === 'qbo') { if (op.data.entityName === 'purchase') { const mapped = applyMapper( @@ -203,6 +274,34 @@ export function bankingLink(ctx: { } const mappers = { + xero: { + account: mapper(zCast>(), zBanking.account, { + id: 'AccountID', + name: 'Name', + }), + category: mapper(zCast>(), zBanking.account, { + id: 'AccountID', + name: 'Name', + }), + bank_transaction: mapper( + zCast>(), + zBanking.transaction, + { + id: 'BankTransactionID', + amount: 'Total', + currency: 'CurrencyCode', + date: 'DateString' as 'Date', // empirically works https://share.cleanshot.com/0c6dlNsF + account_id: 'BankAccount.AccountID', + account_name: 'BankAccount.Name', + merchant_id: 'Contact.ContactID', + merchant_name: 'Contact.Name', + category_id: (t) => t.LineItems[0]?.AccountID ?? '', + description: (t) => t.LineItems[0]?.Description ?? '', + // Don't have data readily available for these... + // category_name is not readily available, only ID is provided + }, + ), + }, // Should be able to have input and output entity types in here also. qbo: { purchase: mapper( diff --git a/verticals/vertical-banking/index.ts b/verticals/vertical-banking/index.ts index e066b5a6..e52470e8 100644 --- a/verticals/vertical-banking/index.ts +++ b/verticals/vertical-banking/index.ts @@ -2,3 +2,4 @@ export * from './banking' // codegen:end export * from './adapters/qbo-adapter' +export * from './adapters/xero-adapter' diff --git a/verticals/vertical-banking/package.json b/verticals/vertical-banking/package.json index e59bf92e..c1a634fe 100644 --- a/verticals/vertical-banking/package.json +++ b/verticals/vertical-banking/package.json @@ -9,6 +9,7 @@ "@usevenice/connector-plaid": "workspace:*", "@usevenice/connector-postgres": "workspace:*", "@usevenice/connector-qbo": "workspace:*", + "@usevenice/connector-xero": "workspace:*", "@usevenice/types": "workspace:*" } }