450 cron events

.github/workflows/crates-postgres.yaml

@@ -78,7 +78,9 @@ jobs:
         run: make -C ./crates/pg test-lint
       - name: crates/pg unit tests
         run: make -C ./crates/pg test-unit
+      - name: start postgres
+        run: make -C migrations start-nobuild
       - name: crates/pg db_tests
-        run: make -C ./crates/pg test-db START_TARGET=start-nobuild
+        run: make -C ./crates/pg test-db
       - name: crates/pg clean up
         run: make -C ./crates/pg stop

.github/workflows/prod-services-deploy.yaml

@@ -27,9 +27,16 @@ jobs:
       - uses: actions/checkout@v4
       - name: mask values
         run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}"
-      - name: deploy services to prod cloud environment
+      - name: deploy lambda services to prod cloud environment
         run: |
-          for app in $(bash scripts/list-deployments.sh | grep -v client | xargs basename -a); do
+          for app_dir in $(bash scripts/list-deployments.sh | grep -v client); do
+            app=$(basename $app_dir)
+            CONF_PATH=$(bash scripts/dir-to-conf-path.sh "$app_dir")
+            DEPLOY_TARGET=$(yq "$CONF_PATH.deploy_target" project.yaml)
+            if [[ "$DEPLOY_TARGET" != "lambda" ]]; then
+              echo "*** skipping $app (deploy_target: $DEPLOY_TARGET)"
+              continue
+            fi
             # tag newly pushed prod images with master merge commit sha
             bash scripts/tag-merge-commit.sh --app-name $app --env prod --env-id ${{ secrets.PROD_ENV_ID }};
             # deploy newly tagged images to prod cloud

.github/workflows/services.yaml

@@ -139,8 +139,10 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - name: stop runner postgres
         run: sudo systemctl stop postgresql
+      - name: start postgres
+        run: make -C migrations start-nobuild
       - name: test database
-        run: make -C crates/pg test-db START_TARGET=start-nobuild
+        run: make -C crates/pg test-db
 
   compile:
     name: compile ${{ matrix.service }}

crates/pg/makefile

@@ -3,8 +3,6 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | gr
 APP_NAME=$(shell basename $(CURDIR))
 PROJECT_CONF_FILE_NAME=project.yaml
 TEST_NAME?=model::integration_tests::
-MIGRATIONS_DIR=$(RELATIVE_PROJECT_ROOT_PATH)/migrations
-START_TARGET?=start
 
 test-unit:
 	cargo test
@@ -25,7 +23,6 @@ watch-db:
 
 test-db:
 	@$(MAKE) get-secrets
-	@$(MAKE) -C $(MIGRATIONS_DIR) $(START_TARGET)
 	@cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/test-reset.sh --set
 	cargo test $(TEST_NAME) --features db_tests -- --test-threads=1
 

crates/types/src/transaction.rs

@@ -84,6 +84,26 @@ impl Transaction {
 }
 
 impl Transaction {
+    pub fn from_rule_instance(
+        rule_instance_id: String,
+        author: Option<String>,
+        author_role: Option<AccountRole>,
+    ) -> Self {
+        Transaction {
+            id: None,
+            rule_instance_id: Some(rule_instance_id),
+            author,
+            author_device_id: None,
+            author_device_latlng: None,
+            author_role,
+            equilibrium_time: None,
+            event_time: None,
+            debitor_first: None,
+            sum_value: "0".to_string(),
+            transaction_items: TransactionItems(vec![]),
+        }
+    }
+
     pub fn new(
         author: String,
         equilibrium_time: Option<TZTime>,
@@ -459,6 +479,46 @@ pub mod tests {
         }
     }
 
+    #[test]
+    fn it_creates_transaction_from_rule_instance_with_author() {
+        let got = Transaction::from_rule_instance(
+            "42".to_string(),
+            Some("StateOfCalifornia".to_string()),
+            Some(AccountRole::Creditor),
+        );
+        assert_eq!(got.rule_instance_id, Some("42".to_string()));
+        assert_eq!(got.author, Some("StateOfCalifornia".to_string()));
+        assert_eq!(got.author_role, Some(AccountRole::Creditor));
+        assert_eq!(got.id, None);
+        assert_eq!(got.author_device_id, None);
+        assert_eq!(got.author_device_latlng, None);
+        assert_eq!(got.equilibrium_time, None);
+        assert_eq!(got.event_time, None);
+        assert_eq!(got.debitor_first, None);
+        assert_eq!(got.sum_value, "0");
+        assert!(got.transaction_items.0.is_empty());
+    }
+
+    #[test]
+    fn it_creates_transaction_from_rule_instance_without_author() {
+        let got = Transaction::from_rule_instance("7".to_string(), None, None);
+        assert_eq!(got.rule_instance_id, Some("7".to_string()));
+        assert_eq!(got.author, None);
+        assert_eq!(got.author_role, None);
+    }
+
+    #[test]
+    fn it_serializes_transaction_from_rule_instance() {
+        let transaction = Transaction::from_rule_instance(
+            "42".to_string(),
+            Some("StateOfCalifornia".to_string()),
+            Some(AccountRole::Creditor),
+        );
+        let json = serde_json::to_string(&transaction).unwrap();
+        let deserialized: Transaction = serde_json::from_str(&json).unwrap();
+        assert_eq!(transaction, deserialized);
+    }
+
     // cadet todo: test remaining branches of get_author_role
     #[test]
     fn it_returns_author_role_found_in_transaction_items() {

docker/go-migrate.Dockerfile

@@ -18,6 +18,7 @@ COPY migrations/schema task/migrations/schema
 COPY migrations/seed task/migrations/seed
 COPY migrations/testseed task/migrations/testseed
 COPY migrations/testseedthresh task/migrations/testseedthresh
+COPY migrations/testseedcron task/migrations/testseedcron
 
 FROM public.ecr.aws/lambda/provided:al2023
 

migrations/go-migrate/migrate.sh

@@ -33,7 +33,7 @@ if [[ -z "$SUBDIRS" ]]; then
 	exit 1
 fi
 
-ALLOWED_SUBDIRS=(schema seed testseed testseedthresh)
+ALLOWED_SUBDIRS=(schema seed testseed testseedthresh testseedcron)
 IFS=',' read -ra SUBDIR_ARGS <<< "$SUBDIRS"
 for s in "${SUBDIR_ARGS[@]}"; do
 	if [[ ! " ${ALLOWED_SUBDIRS[*]} " =~ " ${s} " ]]; then

migrations/makefile

@@ -116,6 +116,34 @@ thresh-up:
 thresh-down:
 	@set -a && source $(ENV_FILE) && set +a && SQL_TYPE=postgresql bash $(RELATIVE_PROJECT_ROOT_PATH)/migrations/go-migrate/migrate.sh --dir $(RELATIVE_PROJECT_ROOT_PATH)/migrations --subdirs testseedthresh --cmd down
 
+cron-up:
+	@set -a && source $(ENV_FILE) && set +a && SQL_TYPE=postgresql bash $(RELATIVE_PROJECT_ROOT_PATH)/migrations/go-migrate/migrate.sh --dir $(RELATIVE_PROJECT_ROOT_PATH)/migrations --subdirs testseedcron --cmd up
+	@$(MAKE) warm-cache
+
+cron-down:
+	@set -a && source $(ENV_FILE) && set +a && SQL_TYPE=postgresql bash $(RELATIVE_PROJECT_ROOT_PATH)/migrations/go-migrate/migrate.sh --dir $(RELATIVE_PROJECT_ROOT_PATH)/migrations --subdirs testseedcron --cmd down
+
+# poll for cron transactions (waits 15 seconds for pg_cron to fire)
+wait-cron:
+	@for i in $$(seq 1 15); do printf '#'; sleep 1; done; echo
+
+# poll redis accumulator until threshold fires (value resets)
+wait-thresh:
+	@key=$$(docker exec mxf-redis-1 redis-cli -a test --no-auth-warning KEYS '*accumulator' 2>/dev/null | head -1); \
+	while [ -z "$$key" ]; do \
+		sleep 2; \
+		key=$$(docker exec mxf-redis-1 redis-cli -a test --no-auth-warning KEYS '*accumulator' 2>/dev/null | head -1); \
+	done; \
+	prev=0; \
+	while true; do \
+		val=$$(docker exec mxf-redis-1 redis-cli -a test --no-auth-warning GET "$$key" 2>/dev/null | tr -d '"'); \
+		if [ -z "$$val" ] || [ "$$val" = "(nil)" ]; then sleep 2; continue; fi; \
+		echo "accumulator: $$val"; \
+		if [ "$$(echo "$$val < $$prev" | bc -l)" = "1" ]; then echo "threshold fired"; break; fi; \
+		prev=$$val; \
+		sleep 2; \
+	done
+
 ###################### env ######################
 
 env:

migrations/schema/000010_notify.down.sql

@@ -0,0 +1 @@
+DROP FUNCTION IF EXISTS notify_event;

migrations/schema/000010_notify.up.sql

@@ -0,0 +1,9 @@
+CREATE OR REPLACE FUNCTION notify_event(event_name TEXT, event_id TEXT)
+RETURNS void AS $$
+BEGIN
+  PERFORM pg_notify('event', json_build_object(
+    'event', event_name,
+    'id', event_id
+  )::text);
+END;
+$$ LANGUAGE plpgsql;

migrations/schema/000010_equilibrium.up.sql → migrations/schema/000011_equilibrium.up.sql

@@ -33,7 +33,7 @@ CREATE INDEX redis_name_key_idx ON redis_name(key);
 
 CREATE OR REPLACE FUNCTION notify_equilibrium() RETURNS TRIGGER AS $$
 BEGIN
-  PERFORM pg_notify('equilibrium', NEW.id::text);
+  PERFORM notify_event('equilibrium', NEW.id::text);
   RETURN NEW;
 END;
 $$ LANGUAGE plpgsql;

migrations/schema/000012_cron.down.sql

@@ -0,0 +1,2 @@
+DROP FUNCTION IF EXISTS notify_cron;
+DROP EXTENSION IF EXISTS pg_cron;

migrations/schema/000012_cron.up.sql

@@ -0,0 +1,8 @@
+CREATE EXTENSION IF NOT EXISTS pg_cron;
+
+CREATE OR REPLACE FUNCTION notify_cron(instance_id INT)
+RETURNS void AS $$
+BEGIN
+  PERFORM notify_event('cron', instance_id::text);
+END;
+$$ LANGUAGE plpgsql;

migrations/testseedcron/000001_interest.down.sql

@@ -0,0 +1,31 @@
+SELECT cron.unschedule('EnergyCoInterest');
+
+DELETE FROM approval WHERE transaction_item_id IN (
+  SELECT id FROM transaction_item WHERE rule_instance_id IN (
+    SELECT id FROM transaction_item_rule_instance WHERE transaction_rule_instance_id = (
+      SELECT id FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoInterest'
+    )
+  )
+);
+
+DELETE FROM transaction_item WHERE rule_instance_id IN (
+  SELECT id FROM transaction_item_rule_instance WHERE transaction_rule_instance_id = (
+    SELECT id FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoInterest'
+  )
+);
+
+DELETE FROM transaction WHERE id NOT IN (SELECT DISTINCT transaction_id FROM transaction_item);
+
+DELETE FROM approval WHERE rule_instance_id IN (
+  SELECT id FROM approval_rule_instance
+  WHERE rule_instance_name IN ('ApproveInterestEnergyCo', 'ApproveInterestJoeCarter')
+);
+
+DELETE FROM approval_rule_instance WHERE rule_instance_name IN ('ApproveInterestEnergyCo', 'ApproveInterestJoeCarter');
+
+DELETE FROM transaction_item_rule_instance
+WHERE transaction_rule_instance_id = (
+  SELECT id FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoInterest'
+);
+
+DELETE FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoInterest';

migrations/testseedcron/000001_interest.up.sql

@@ -0,0 +1,44 @@
+-- cron debt: EnergyCo pays interest to JoeCarter every 10 seconds
+--
+-- same debitor, creditor and amount as cron equity (000002_dividend)
+-- modigliani-miller: "debt" and "equity" are just rule instance names
+-- when the underlying value transfer is identical
+
+insert into transaction_rule_instance
+  (rule_name, rule_instance_name, author, author_role, cron)
+values
+  ('createTransaction', 'EnergyCoInterest', 'EnergyCo', 'debitor', '10 seconds');
+
+-- transaction_item_rule_instance: linked line item
+insert into transaction_item_rule_instance
+  (rule_name, rule_instance_name, account_role, account_name,
+   item_id, price, quantity, variable_values, transaction_rule_instance_id)
+values
+  ('addTransactionItem', 'EnergyCoInterest', 'debitor', 'EnergyCo',
+   '1,000 x 1% monthly interest', 10.000, 1, '{ "EnergyCo", "JoeCarter" }',
+   (select id from transaction_rule_instance where rule_instance_name = 'EnergyCoInterest'));
+
+-- approval_rule_instance: AaronHill approves EnergyCo debits (EnergyCo owner)
+insert into approval_rule_instance
+  (rule_name, rule_instance_name, account_role, account_name, variable_values)
+values
+  ('approveItemBetweenAccounts', 'ApproveInterestEnergyCo', 'debitor', 'AaronHill',
+   '{ "EnergyCo", "JoeCarter", "1,000 x 1% monthly interest", "debitor", "AaronHill" }');
+
+-- approval_rule_instance: JoeCarter approves credits (self-owner)
+insert into approval_rule_instance
+  (rule_name, rule_instance_name, account_role, account_name, variable_values)
+values
+  ('approveItemBetweenAccounts', 'ApproveInterestJoeCarter', 'creditor', 'JoeCarter',
+   '{ "EnergyCo", "JoeCarter", "1,000 x 1% monthly interest", "creditor", "JoeCarter" }');
+
+-- schedule pg_cron job and store jobid
+UPDATE transaction_rule_instance
+SET cron_job_id = cron.schedule(
+  'EnergyCoInterest',
+  '10 seconds',
+  $$SELECT notify_cron(
+    (SELECT id FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoInterest')
+  )$$
+)
+WHERE rule_instance_name = 'EnergyCoInterest';

migrations/testseedcron/000002_dividend.down.sql

@@ -0,0 +1,31 @@
+SELECT cron.unschedule('EnergyCoDividend');
+
+DELETE FROM approval WHERE transaction_item_id IN (
+  SELECT id FROM transaction_item WHERE rule_instance_id IN (
+    SELECT id FROM transaction_item_rule_instance WHERE transaction_rule_instance_id = (
+      SELECT id FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoDividend'
+    )
+  )
+);
+
+DELETE FROM transaction_item WHERE rule_instance_id IN (
+  SELECT id FROM transaction_item_rule_instance WHERE transaction_rule_instance_id = (
+    SELECT id FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoDividend'
+  )
+);
+
+DELETE FROM transaction WHERE id NOT IN (SELECT DISTINCT transaction_id FROM transaction_item);
+
+DELETE FROM approval WHERE rule_instance_id IN (
+  SELECT id FROM approval_rule_instance
+  WHERE rule_instance_name IN ('ApproveEnergyDividendEnergyCo', 'ApproveEnergyDividendJoeCarter')
+);
+
+DELETE FROM approval_rule_instance WHERE rule_instance_name IN ('ApproveEnergyDividendEnergyCo', 'ApproveEnergyDividendJoeCarter');
+
+DELETE FROM transaction_item_rule_instance
+WHERE transaction_rule_instance_id = (
+  SELECT id FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoDividend'
+);
+
+DELETE FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoDividend';

migrations/testseedcron/000002_dividend.up.sql

@@ -0,0 +1,44 @@
+-- cron equity: EnergyCo pays dividend to JoeCarter every 10 seconds
+--
+-- same debitor, creditor and amount as cron debt (000001_interest)
+-- modigliani-miller: "debt" and "equity" are just rule instance names
+-- when the underlying value transfer is identical
+
+insert into transaction_rule_instance
+  (rule_name, rule_instance_name, author, author_role, cron)
+values
+  ('createTransaction', 'EnergyCoDividend', 'EnergyCo', 'debitor', '10 seconds');
+
+-- transaction_item_rule_instance: linked line item
+insert into transaction_item_rule_instance
+  (rule_name, rule_instance_name, account_role, account_name,
+   item_id, price, quantity, variable_values, transaction_rule_instance_id)
+values
+  ('addTransactionItem', 'EnergyCoDividend', 'debitor', 'EnergyCo',
+   'monthly 10.000 dividend', 10.000, 1, '{ "EnergyCo", "JoeCarter" }',
+   (select id from transaction_rule_instance where rule_instance_name = 'EnergyCoDividend'));
+
+-- approval_rule_instance: AaronHill approves EnergyCo debits (EnergyCo owner)
+insert into approval_rule_instance
+  (rule_name, rule_instance_name, account_role, account_name, variable_values)
+values
+  ('approveItemBetweenAccounts', 'ApproveEnergyDividendEnergyCo', 'debitor', 'AaronHill',
+   '{ "EnergyCo", "JoeCarter", "monthly 10.000 dividend", "debitor", "AaronHill" }');
+
+-- approval_rule_instance: JoeCarter approves credits (self-owner)
+insert into approval_rule_instance
+  (rule_name, rule_instance_name, account_role, account_name, variable_values)
+values
+  ('approveItemBetweenAccounts', 'ApproveEnergyDividendJoeCarter', 'creditor', 'JoeCarter',
+   '{ "EnergyCo", "JoeCarter", "monthly 10.000 dividend", "creditor", "JoeCarter" }');
+
+-- schedule pg_cron job and store jobid
+UPDATE transaction_rule_instance
+SET cron_job_id = cron.schedule(
+  'EnergyCoDividend',
+  '10 seconds',
+  $$SELECT notify_cron(
+    (SELECT id FROM transaction_rule_instance WHERE rule_instance_name = 'EnergyCoDividend')
+  )$$
+)
+WHERE rule_instance_name = 'EnergyCoDividend';

migrations/testseedthresh/000001_dividend.down.sql

@@ -1,6 +1,6 @@
 delete from approval where transaction_item_id in (select ti.id from transaction_item ti where ti.rule_instance_id in (select id from transaction_item_rule_instance where rule_instance_name = 'GroceryCoProfitDividend'));
 delete from transaction_item where rule_instance_id in (select id from transaction_item_rule_instance where rule_instance_name = 'GroceryCoProfitDividend');
 delete from transaction where id not in (select distinct transaction_id from transaction_item);
-delete from approval_rule_instance where rule_instance_name in ('ApproveDividendGroceryCo', 'ApproveDividendTimeCo');
+delete from approval_rule_instance where rule_instance_name in ('ApproveGroceryDividendGroceryCo', 'ApproveGroceryDividendJoeCarter');
 delete from transaction_item_rule_instance where rule_instance_name = 'GroceryCoProfitDividend';
 delete from transaction_rule_instance where rule_instance_name = 'GroceryCoProfitDividend';