Move the rest of the syncing protobuf code to the CRDT package (#1217)

.prettierignore

+sync_pb.*

packages/crdt/.gitignore

+bin/protoc-gen-js
+bin/protoc

packages/loot-core/src/server/sync/README.md → packages/crdt/README.md

-# protobuf
+# `@actual-app/crdt`
 
-We use [protobuf](https://developers.google.com/protocol-buffers/) to encode messages as binary data to send across the network.
+This package contains the core CRDT logic that enables Actual’s syncing. It is shared between the client and server. We may or may not follow semver when updating this package; any usage of it outside Actual is undocumented and at your own risk.
 
-## Generating protobuf
+## protobuf
 
-The protobuf is generated by using the [protoc](https://github.com/protocolbuffers/protobuf) compiler.
+We use [protobuf](https://developers.google.com/protocol-buffers/) to encode messages as binary data to send across the network.
 
-This can be installed by downloading one of the [pre-built binaries](https://github.com/protocolbuffers/protobuf/releases/) and placing it in your `$PATH`. The version used to build the current protobuf is [v3.20.1](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.1).
+### Generating protobuf
 
-Once installed, the protobuf can be generated by running the below commands.
+The protobuf is generated by using the [protoc](https://github.com/protocolbuffers/protobuf) compiler.
 
-```bash
-cd packages/loot-core
+This can be installed by downloading one of the [pre-built binaries](https://github.com/protocolbuffers/protobuf/releases/) and placing it in your `$PATH`. The version used to build the current protobuf is [v3.20.1](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.1). You’ll also need to [download the latest version of `protoc-gen-js`](https://github.com/protocolbuffers/protobuf-javascript/releases/latest). For convenience, you can put both of these binaries in `./bin`.
 
-protoc --plugin="protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts" \
-    --ts_opt=esModuleInterop=true \
-    --ts_out="src/server/sync/proto" \
-    --js_out=import_style=commonjs,binary:src/server/sync/proto \
-    --proto_path=src/server/sync \
-    sync.proto
-```
+Once installed, the protobuf can be generated by running `./bin/generate-proto`.
 
 However there is one very important thing to remember! The default output includes this near the top:
 

packages/crdt/bin/generate-proto

+#!/bin/bash
+
+cd "$(dirname "$(dirname "$0")")"
+
+if ! [ -x "$(command -v protoc)" ]; then
+  echo 'Error: protoc is not installed. See the readme for installation instructions.' >&2
+  exit 1
+fi
+
+export PATH="$PWD/bin:$PATH"
+
+protoc --plugin="protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts" \
+  --ts_opt=esModuleInterop=true \
+  --ts_out="src/proto" \
+  --js_out=import_style=commonjs,binary:src/proto \
+  --proto_path=src/proto \
+  sync.proto
+
+../../node_modules/.bin/prettier --write src/proto/*.d.ts
+
+echo 'One more step! Find the `var global = ...` declaration in src/proto/sync_pb.js and change it to `var global = globalThis;`'

packages/crdt/package.json

@@ -10,6 +10,7 @@
   ],
   "scripts": {
     "build:node": "tsc --p tsconfig.dist.json",
+    "proto:generate": "./bin/generate-proto",
     "build": "rm -rf dist && yarn run build:node && cp src/proto/sync_pb.d.ts dist/src/proto/",
     "test": "jest -c jest.config.js"
   },
@@ -22,6 +23,7 @@
     "@types/jest": "^27.5.0",
     "jest": "^27.0.0",
     "ts-jest": "^27.0.0",
+    "ts-protoc-gen": "^0.15.0",
     "typescript": "^5.0.2"
   }
 }

packages/crdt/src/proto/sync_pb.d.ts

@@ -20,11 +20,11 @@ export class EncryptedData extends jspb.Message {
   setData(value: Uint8Array | string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): EncryptedDataAsObject;
+  toObject(includeInstance?: boolean): EncryptedData.AsObject;
   static toObject(
     includeInstance: boolean,
     msg: EncryptedData,
-  ): EncryptedDataAsObject;
+  ): EncryptedData.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -40,11 +40,13 @@ export class EncryptedData extends jspb.Message {
   ): EncryptedData;
 }
 
-export type EncryptedDataAsObject = {
-  iv: Uint8Array | string;
-  authtag: Uint8Array | string;
-  data: Uint8Array | string;
-};
+export namespace EncryptedData {
+  export type AsObject = {
+    iv: Uint8Array | string;
+    authtag: Uint8Array | string;
+    data: Uint8Array | string;
+  };
+}
 
 export class Message extends jspb.Message {
   getDataset(): string;
@@ -60,8 +62,8 @@ export class Message extends jspb.Message {
   setValue(value: string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): MessageAsObject;
-  static toObject(includeInstance: boolean, msg: Message): MessageAsObject;
+  toObject(includeInstance?: boolean): Message.AsObject;
+  static toObject(includeInstance: boolean, msg: Message): Message.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -77,12 +79,14 @@ export class Message extends jspb.Message {
   ): Message;
 }
 
-export type MessageAsObject = {
-  dataset: string;
-  row: string;
-  column: string;
-  value: string;
-};
+export namespace Message {
+  export type AsObject = {
+    dataset: string;
+    row: string;
+    column: string;
+    value: string;
+  };
+}
 
 export class MessageEnvelope extends jspb.Message {
   getTimestamp(): string;
@@ -97,11 +101,11 @@ export class MessageEnvelope extends jspb.Message {
   setContent(value: Uint8Array | string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): MessageEnvelopeAsObject;
+  toObject(includeInstance?: boolean): MessageEnvelope.AsObject;
   static toObject(
     includeInstance: boolean,
     msg: MessageEnvelope,
-  ): MessageEnvelopeAsObject;
+  ): MessageEnvelope.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -117,11 +121,13 @@ export class MessageEnvelope extends jspb.Message {
   ): MessageEnvelope;
 }
 
-export type MessageEnvelopeAsObject = {
-  timestamp: string;
-  isencrypted: boolean;
-  content: Uint8Array | string;
-};
+export namespace MessageEnvelope {
+  export type AsObject = {
+    timestamp: string;
+    isencrypted: boolean;
+    content: Uint8Array | string;
+  };
+}
 
 export class SyncRequest extends jspb.Message {
   clearMessagesList(): void;
@@ -142,11 +148,11 @@ export class SyncRequest extends jspb.Message {
   setSince(value: string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): SyncRequestAsObject;
+  toObject(includeInstance?: boolean): SyncRequest.AsObject;
   static toObject(
     includeInstance: boolean,
     msg: SyncRequest,
-  ): SyncRequestAsObject;
+  ): SyncRequest.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -162,13 +168,15 @@ export class SyncRequest extends jspb.Message {
   ): SyncRequest;
 }
 
-export type SyncRequestAsObject = {
-  messagesList: Array<MessageEnvelopeAsObject>;
-  fileid: string;
-  groupid: string;
-  keyid: string;
-  since: string;
-};
+export namespace SyncRequest {
+  export type AsObject = {
+    messagesList: Array<MessageEnvelope.AsObject>;
+    fileid: string;
+    groupid: string;
+    keyid: string;
+    since: string;
+  };
+}
 
 export class SyncResponse extends jspb.Message {
   clearMessagesList(): void;
@@ -180,11 +188,11 @@ export class SyncResponse extends jspb.Message {
   setMerkle(value: string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): SyncResponseAsObject;
+  toObject(includeInstance?: boolean): SyncResponse.AsObject;
   static toObject(
     includeInstance: boolean,
     msg: SyncResponse,
-  ): SyncResponseAsObject;
+  ): SyncResponse.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -200,7 +208,9 @@ export class SyncResponse extends jspb.Message {
   ): SyncResponse;
 }
 
-export type SyncResponseAsObject = {
-  messagesList: Array<MessageEnvelopeAsObject>;
-  merkle: string;
-};
+export namespace SyncResponse {
+  export type AsObject = {
+    messagesList: Array<MessageEnvelope.AsObject>;
+    merkle: string;
+  };
+}

packages/loot-core/package.json

@@ -72,7 +72,6 @@
     "throttleit": "^1.0.0",
     "ts-jest": "^27.0.0",
     "ts-node": "^10.7.0",
-    "ts-protoc-gen": "^0.15.0",
     "typescript": "^4.6.4",
     "uuid": "3.3.2",
     "webpack": "^5.86.0",

upcoming-release-notes/1217.md

+---
+category: Maintenance
+authors: [j-f1]
+---
+
+Move the rest of the syncing protobuf code to the CRDT package

yarn.lock

@@ -32,6 +32,7 @@ __metadata:
     jest: ^27.0.0
     murmurhash: ^0.0.2
     ts-jest: ^27.0.0
+    ts-protoc-gen: ^0.15.0
     typescript: ^5.0.2
     uuid: 3.3.2
   languageName: unknown
@@ -12380,7 +12381,6 @@ __metadata:
     throttleit: ^1.0.0
     ts-jest: ^27.0.0
     ts-node: ^10.7.0
-    ts-protoc-gen: ^0.15.0
     typescript: ^4.6.4
     uuid: 3.3.2
     webpack: ^5.86.0