Merge pull request #120 from HannesOberreiter/beta

Beta

db/migrations/20230901092251_observations.js

@@ -7,7 +7,7 @@ export const up = function (knex) {
     t.increments('id').primary().unsigned();
     t.string('taxa', 45).index();
     t.string('external_service', 45);
-    t.integer('external_id').unsigned().index();
+    t.integer('external_id').unsigned().nullable().index();
     t.point('location');
     t.json('data');
 

db/migrations/20230902201815_alter_observations.js

@@ -0,0 +1,19 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export const up = function (knex) {
+  return knex.schema.alterTable('observations', (t) => {
+    t.string('external_uuid', 65);
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export const down = function (knex) {
+  return knex.schema.alterTable('observations', (t) => {
+    t.dropColumn('external_uuid');
+  });
+};

src/api/controllers/public.controller.ts

@@ -0,0 +1,75 @@
+import { FastifyRequest, FastifyReply } from 'fastify';
+import { Observation } from '../models/observation.model.js';
+import { raw } from 'objection';
+import { RedisServer } from '../../servers/redis.server.js';
+
+export default class PublicController {
+  static async getVelutinaObservationsRecent(
+    _req: FastifyRequest,
+    reply: FastifyReply,
+  ) {
+    reply.header('Cache-Control', 'public, max-age=21600');
+
+    const redis = RedisServer.client;
+    const cached = await redis.get('cache:velutinaObservationsRecent');
+    if (cached) {
+      return cached;
+    }
+
+    const start = new Date(Date.now() - 1000 * 60 * 60 * 24 * 182);
+    const end = new Date(Date.now());
+    const result = await Observation.query()
+      .select(
+        'location',
+        raw('JSON_EXTRACT(data, "$.uri") as uri'),
+        'observed_at',
+      )
+      .whereBetween('observed_at', [start, end]);
+
+    redis.set(
+      'cache:velutinaObservationsRecent',
+      JSON.stringify(result),
+      'EX',
+      21600,
+    );
+
+    return result;
+  }
+
+  static async getVelutinaObservationsStats(
+    _req: FastifyRequest,
+    reply: FastifyReply,
+  ) {
+    reply.header('Cache-Control', 'public, max-age=21600');
+    const res = await Observation.query().count('id as count');
+    return res[0];
+  }
+
+  static async getVelutinaObservationsArray(
+    _req: FastifyRequest,
+    reply: FastifyReply,
+  ) {
+    reply.header('Cache-Control', 'public, max-age=21600');
+
+    const redis = RedisServer.client;
+    const cached = await redis.get('cache:velutinaObservationsArray');
+    if (cached) {
+      return cached;
+    }
+
+    const result = await Observation.query().select('location');
+    const res = [];
+    for (const r of result) {
+      res.push([(r as any).location.x, (r as any).location.y]);
+    }
+
+    redis.set(
+      'cache:velutinaObservationsArray',
+      JSON.stringify(res),
+      'EX',
+      21600,
+    );
+
+    return res;
+  }
+}

src/api/controllers/service.controller.ts

@@ -131,13 +131,4 @@ export default class ServiceController {
 
     return { ...result, savedTokens, savedQuestions };
   }
-
-  static async getObservations(req: FastifyRequest, reply: FastifyReply) {
-    const start = new Date(Date.now() - 1000 * 60 * 60 * 24 * 182);
-    const end = new Date(Date.now());
-    const result = await Observation.query()
-      .select('*', raw('ST_AsGeoJSON(location) as location'))
-      .whereBetween('observed_at', [start, end]);
-    return result;
-  }
 }

src/api/models/observation.model.ts

@@ -4,12 +4,14 @@ import { ExtModel } from './base.model.js';
 export class Observation extends ExtModel {
   id!: number;
   taxa!: 'Vespa velutina';
-  external_service!: 'iNaturalist';
-  external_id!: string;
+  external_service!: 'iNaturalist' | 'PatriNat';
+  external_id!: number;
+  external_uuid!: string;
   location!: {
     lat: number;
     lng: number;
   };
+  data!: any;
 
   observed_at!: string;
 
@@ -18,18 +20,13 @@ export class Observation extends ExtModel {
 
   static jsonSchema = {
     type: 'object',
-    required: [
-      'taxa',
-      'external_service',
-      'external_id',
-      'location',
-      'observed_at',
-    ],
+    required: ['taxa', 'external_service', 'location', 'observed_at'],
     properties: {
       id: { type: 'integer' },
       taxa: { type: 'string', minLength: 1, maxLength: 45 },
       external_service: { type: 'string', minLength: 1, maxLength: 45 },
       external_id: { type: 'integer' },
+      external_uuid: { type: 'string' },
       location: { type: 'object' },
       data: { type: 'object' },
 
@@ -46,7 +43,6 @@ export class Observation extends ExtModel {
       location.lat,
       location.lng,
     ]);
-    console.log(rawLocation);
     formattedJson.location = rawLocation;
     return formattedJson;
   }

src/api/routes/index.ts

@@ -27,6 +27,7 @@ import v1RearingStep from './v1/rearing_step.route.js';
 import v1FieldSetting from './v1/field_setting.route.js';
 import v1Service from './v1/service.route.js';
 import v1Todo from './v1/todo.route.js';
+import v1Public from './v1/public.route.js';
 
 export default function routes(app: FastifyInstance, _options: any, done: any) {
   app.register(v1Root, {
@@ -134,5 +135,9 @@ export default function routes(app: FastifyInstance, _options: any, done: any) {
     prefix: '/v1/statistic',
   });
 
+  app.register(v1Public, {
+    prefix: '/v1/public',
+  });
+
   done();
 }

src/api/routes/v1/public.route.ts

@@ -0,0 +1,29 @@
+import { FastifyInstance } from 'fastify';
+import { ZodTypeProvider } from 'fastify-type-provider-zod';
+import PublicController from '../../controllers/public.controller.js';
+
+export default function routes(
+  instance: FastifyInstance,
+  _options: any,
+  done: any,
+) {
+  const server = instance.withTypeProvider<ZodTypeProvider>();
+
+  server.get(
+    '/velutina/observations/recent',
+    {},
+    PublicController.getVelutinaObservationsRecent,
+  );
+  server.get(
+    '/velutina/observations/stats',
+    {},
+    PublicController.getVelutinaObservationsStats,
+  );
+  server.get(
+    '/velutina/observations/array',
+    {},
+    PublicController.getVelutinaObservationsArray,
+  );
+
+  done();
+}

src/api/routes/v1/service.route.ts

@@ -68,13 +68,5 @@ export default function routes(
     ServiceController.askWizBee,
   );
 
-  server.get(
-    '/observation',
-    {
-      preHandler: Guard.authorize([ROLES.admin, ROLES.user, ROLES.read]),
-    },
-    ServiceController.getObservations,
-  );
-
   done();
 }

src/api/utils/velutina.util.ts

@@ -1,6 +1,77 @@
 import { Observation } from '../models/observation.model.js';
 
 export async function fetchObservations() {
+  const inat = fetchInat();
+  const patriNat = fetchPatriNat();
+  const [inatResult, patriNatResult] = await Promise.all([inat, patriNat]);
+  return { iNaturalist: inatResult, patriNat: patriNatResult };
+}
+
+type ObservationPatriNat = {
+  nom_station: string;
+  loc_lat: string;
+  loc_long: string;
+  observateur: string;
+  validation: string;
+  commentaires_validation: string;
+};
+
+async function fetchPatriNat() {
+  // Example Input: Rome07072015033744
+  function dateExtract(t: string) {
+    const e = t.substring(t.length - 14),
+      day = e.substring(0, 2),
+      month = e.substring(2, 4),
+      year = e.substring(4, 8);
+    return new Date(year + '-' + month + '-' + day);
+  }
+  const result = await fetch(
+    'https://frelonasiatique.mnhn.fr/wp-content/plugins/spn_csv_exporter/widgetVisualisateur/visuaMapDisplayController.php',
+    {
+      headers: {
+        Referer: 'https://www.btree.at/',
+      },
+      method: 'GET',
+    },
+  );
+  const data = await result.json();
+  const observations = [];
+  if (!data.records) {
+    return { newObservations: 0 };
+  }
+
+  const oldRecords = await Observation.query()
+    .select('external_uuid')
+    .where('external_service', 'PatriNat');
+
+  for (const observation of data.records as ObservationPatriNat[]) {
+    if (observation.validation != 'Validé') continue;
+    const date = dateExtract(observation.nom_station);
+    if (oldRecords.find((o) => o.external_uuid === observation.nom_station))
+      continue;
+    observation['uri'] = 'https://frelonasiatique.mnhn.fr/visualisateur';
+    observations.push({
+      external_uuid: observation.nom_station,
+      external_service: 'PatriNat',
+      observed_at: date.toISOString(),
+      location: {
+        lat: Number(observation.loc_lat),
+        lng: Number(observation.loc_long),
+      },
+      taxa: 'Vespa velutina',
+      data: observation,
+    });
+  }
+
+  if (observations.length === 0) return { newObservations: 0 };
+
+  await Observation.query().insertGraph(observations);
+
+  //await Observation.query().insertGraph(observations);
+  return { newObservations: observations.length };
+}
+
+async function fetchInat() {
   const fields =
     '(id:!t,uri:!t,quality_grade:!t,time_observed_at:!t,location:!t,taxon:(id:!t))';
 

src/app.bootstrap.ts

@@ -30,9 +30,9 @@ const application = new Application();
 const httpServer = new HTTPServer(application.app);
 httpServer.start();
 
-closeWithGrace({ delay: 5000 }, async function ({ signal, err, manual }) {
-  if (err) {
-    console.error(err);
+closeWithGrace({ delay: 5000 }, async function (res) {
+  if (res.err) {
+    console.error(res.err);
   }
   await gracefulShutdown();
 });

src/services/cron.service.ts

@@ -51,7 +51,9 @@ export class Cron {
           this.Logging(await reminderDeletion());
           this.Logging(await reminderVIS());
           this.Logging(await reminderPremium());
-          fetchObservations().then((res) => this.Logging(res));
+          fetchObservations()
+            .then((res) => this.Logging(res))
+            .catch((e) => this.logger.log('error', e, { label: 'CronJob' }));
         } catch (e) {
           this.logger.log('error', e, { label: 'CronJob' });
         }