Rewrite fetcher (#1792)

* Use new fetcher implementation for post/comment

* rewrite person fetch to use new fetcher

* rewrite community to use new fetcher

* rename new_fetcher to dereference_object_id

* make ObjectId a newtype

* handle deletion in new fetcher

* rewrite apub object search to be generic

* move upsert() method out of ApubObject trait

* simplify ObjectId::new (and fix clippy)

api_tests/src/post.spec.ts

@@ -135,7 +135,8 @@ test('Update a post', async () => {
 test('Sticky a post', async () => {
   let postRes = await createPost(alpha, betaCommunity.community.id);
 
-  let stickiedPostRes = await stickyPost(alpha, true, postRes.post_view.post);
+  let betaPost1 = (await resolvePost(beta, postRes.post_view.post)).post;
+  let stickiedPostRes = await stickyPost(beta, true, betaPost1.post);
   expect(stickiedPostRes.post_view.post.stickied).toBe(true);
 
   // Make sure that post is stickied on beta
@@ -145,7 +146,7 @@ test('Sticky a post', async () => {
   expect(betaPost.post.stickied).toBe(true);
 
   // Unsticky a post
-  let unstickiedPost = await stickyPost(alpha, false, postRes.post_view.post);
+  let unstickiedPost = await stickyPost(beta, false, betaPost1.post);
   expect(unstickiedPost.post_view.post.stickied).toBe(false);
 
   // Make sure that post is unstickied on beta

crates/api/src/comment.rs

@@ -13,7 +13,7 @@ use lemmy_apub::{
     undo_vote::UndoVote,
     vote::{Vote, VoteType},
   },
-  PostOrComment,
+  fetcher::post_or_comment::PostOrComment,
 };
 use lemmy_db_queries::{source::comment::Comment_, Likeable, Saveable};
 use lemmy_db_schema::{source::comment::*, LocalUserId};

crates/api/src/post.rs

@@ -19,7 +19,7 @@ use lemmy_apub::{
     },
     CreateOrUpdateType,
   },
-  PostOrComment,
+  fetcher::post_or_comment::PostOrComment,
 };
 use lemmy_db_queries::{source::post::Post_, Crud, Likeable, Saveable};
 use lemmy_db_schema::source::{moderator::*, post::*};

crates/api/src/site.rs

@@ -1,6 +1,7 @@
 use crate::Perform;
 use actix_web::web::Data;
 use anyhow::Context;
+use diesel::NotFound;
 use lemmy_api_common::{
   blocking,
   build_federated_instances,
@@ -9,24 +10,32 @@ use lemmy_api_common::{
   is_admin,
   site::*,
 };
-use lemmy_apub::{build_actor_id_from_shortname, fetcher::search::search_by_apub_id, EndpointType};
+use lemmy_apub::{
+  build_actor_id_from_shortname,
+  fetcher::search::{search_by_apub_id, SearchableObjects},
+  EndpointType,
+};
 use lemmy_db_queries::{
   from_opt_str_to_opt_enum,
   source::site::Site_,
   Crud,
+  DbPool,
   DeleteableOrRemoveable,
   ListingType,
   SearchType,
   SortType,
 };
-use lemmy_db_schema::source::{moderator::*, site::Site};
+use lemmy_db_schema::{
+  source::{moderator::*, site::Site},
+  PersonId,
+};
 use lemmy_db_views::{
-  comment_view::CommentQueryBuilder,
-  post_view::PostQueryBuilder,
+  comment_view::{CommentQueryBuilder, CommentView},
+  post_view::{PostQueryBuilder, PostView},
   site_view::SiteView,
 };
 use lemmy_db_views_actor::{
-  community_view::CommunityQueryBuilder,
+  community_view::{CommunityQueryBuilder, CommunityView},
   person_view::{PersonQueryBuilder, PersonViewSafe},
 };
 use lemmy_db_views_moderator::{
@@ -376,11 +385,52 @@ impl Perform for ResolveObject {
     _websocket_id: Option<ConnectionId>,
   ) -> Result<ResolveObjectResponse, LemmyError> {
     let local_user_view = get_local_user_view_from_jwt_opt(&self.auth, context.pool()).await?;
-    let res = search_by_apub_id(&self.q, local_user_view, context)
+    let res = search_by_apub_id(&self.q, context)
       .await
       .map_err(|_| ApiError::err("couldnt_find_object"))?;
-    Ok(res)
+    convert_response(res, local_user_view.map(|l| l.person.id), context.pool())
+      .await
+      .map_err(|_| ApiError::err("couldnt_find_object").into())
+  }
+}
+
+async fn convert_response(
+  object: SearchableObjects,
+  user_id: Option<PersonId>,
+  pool: &DbPool,
+) -> Result<ResolveObjectResponse, LemmyError> {
+  let removed_or_deleted;
+  let mut res = ResolveObjectResponse {
+    comment: None,
+    post: None,
+    community: None,
+    person: None,
+  };
+  use SearchableObjects::*;
+  match object {
+    Person(p) => {
+      removed_or_deleted = p.deleted;
+      res.person = Some(blocking(pool, move |conn| PersonViewSafe::read(conn, p.id)).await??)
+    }
+    Community(c) => {
+      removed_or_deleted = c.deleted || c.removed;
+      res.community =
+        Some(blocking(pool, move |conn| CommunityView::read(conn, c.id, user_id)).await??)
+    }
+    Post(p) => {
+      removed_or_deleted = p.deleted || p.removed;
+      res.post = Some(blocking(pool, move |conn| PostView::read(conn, p.id, user_id)).await??)
+    }
+    Comment(c) => {
+      removed_or_deleted = c.deleted || c.removed;
+      res.comment = Some(blocking(pool, move |conn| CommentView::read(conn, c.id, user_id)).await??)
+    }
+  };
+  // if the object was deleted from database, dont return it
+  if removed_or_deleted {
+    return Err(NotFound {}.into());
   }
+  Ok(res)
 }
 
 #[async_trait::async_trait(?Send)]

crates/api_crud/src/comment/create.rs

@@ -15,9 +15,9 @@ use lemmy_apub::{
     voting::vote::{Vote, VoteType},
     CreateOrUpdateType,
   },
+  fetcher::post_or_comment::PostOrComment,
   generate_apub_endpoint,
   EndpointType,
-  PostOrComment,
 };
 use lemmy_db_queries::{source::comment::Comment_, Crud, Likeable};
 use lemmy_db_schema::source::comment::*;

crates/api_crud/src/post/create.rs

@@ -13,9 +13,9 @@ use lemmy_apub::{
     voting::vote::{Vote, VoteType},
     CreateOrUpdateType,
   },
+  fetcher::post_or_comment::PostOrComment,
   generate_apub_endpoint,
   EndpointType,
-  PostOrComment,
 };
 use lemmy_db_queries::{source::post::Post_, Crud, Likeable};
 use lemmy_db_schema::source::post::*;

crates/apub/src/activities/comment/create_or_update.rs

@@ -10,6 +10,7 @@ use crate::{
   },
   activity_queue::send_to_community_new,
   extensions::context::lemmy_context,
+  fetcher::object_id::ObjectId,
   objects::{comment::Note, FromApub, ToApub},
   ActorType,
 };
@@ -26,7 +27,7 @@ use url::Url;
 #[derive(Clone, Debug, Deserialize, Serialize, ActivityFields)]
 #[serde(rename_all = "camelCase")]
 pub struct CreateOrUpdateComment {
-  actor: Url,
+  actor: ObjectId<Person>,
   to: [PublicUrl; 1],
   object: Note,
   cc: Vec<Url>,
@@ -60,7 +61,7 @@ impl CreateOrUpdateComment {
     let maa = collect_non_local_mentions(comment, &community, context).await?;
 
     let create_or_update = CreateOrUpdateComment {
-      actor: actor.actor_id(),
+      actor: ObjectId::new(actor.actor_id()),
       to: [PublicUrl::Public],
       object: comment.to_apub(context.pool()).await?,
       cc: maa.ccs,
@@ -84,11 +85,11 @@ impl ActivityHandler for CreateOrUpdateComment {
     request_counter: &mut i32,
   ) -> Result<(), LemmyError> {
     let community = extract_community(&self.cc, context, request_counter).await?;
+    let community_id = ObjectId::new(community.actor_id());
 
     verify_activity(self)?;
-    verify_person_in_community(&self.actor, &community.actor_id(), context, request_counter)
-      .await?;
-    verify_domains_match(&self.actor, self.object.id_unchecked())?;
+    verify_person_in_community(&self.actor, &community_id, context, request_counter).await?;
+    verify_domains_match(self.actor.inner(), self.object.id_unchecked())?;
     // TODO: should add a check that the correct community is in cc (probably needs changes to
     //       comment deserialization)
     self.object.verify(context, request_counter).await?;
@@ -100,7 +101,8 @@ impl ActivityHandler for CreateOrUpdateComment {
     context: &LemmyContext,
     request_counter: &mut i32,
   ) -> Result<(), LemmyError> {
-    let comment = Comment::from_apub(&self.object, context, &self.actor, request_counter).await?;
+    let comment =
+      Comment::from_apub(&self.object, context, self.actor.inner(), request_counter).await?;
     let recipients = get_notif_recipients(&self.actor, &comment, context, request_counter).await?;
     let notif_type = match self.kind {
       CreateOrUpdateType::Create => UserOperationCrud::CreateComment,

crates/apub/src/activities/comment/mod.rs

@@ -1,4 +1,4 @@
-use crate::{fetcher::person::get_or_fetch_and_upsert_person, ActorType};
+use crate::{fetcher::object_id::ObjectId, ActorType};
 use activitystreams::{
   base::BaseExt,
   link::{LinkExt, Mention},
@@ -26,14 +26,14 @@ use url::Url;
 pub mod create_or_update;
 
 async fn get_notif_recipients(
-  actor: &Url,
+  actor: &ObjectId<Person>,
   comment: &Comment,
   context: &LemmyContext,
   request_counter: &mut i32,
 ) -> Result<Vec<LocalUserId>, LemmyError> {
   let post_id = comment.post_id;
   let post = blocking(context.pool(), move |conn| Post::read(conn, post_id)).await??;
-  let actor = get_or_fetch_and_upsert_person(actor, context, request_counter).await?;
+  let actor = actor.dereference(context, request_counter).await?;
 
   // Note:
   // Although mentions could be gotten from the post tags (they are included there), or the ccs,
@@ -76,14 +76,17 @@ pub async fn collect_non_local_mentions(
   for mention in &mentions {
     // TODO should it be fetching it every time?
     if let Ok(actor_id) = fetch_webfinger_url(mention, context.client()).await {
+      let actor_id: ObjectId<Person> = ObjectId::new(actor_id);
       debug!("mention actor_id: {}", actor_id);
       addressed_ccs.push(actor_id.to_owned().to_string().parse()?);
 
-      let mention_person = get_or_fetch_and_upsert_person(&actor_id, context, &mut 0).await?;
+      let mention_person = actor_id.dereference(context, &mut 0).await?;
       inboxes.push(mention_person.get_shared_inbox_or_inbox_url());
 
       let mut mention_tag = Mention::new();
-      mention_tag.set_href(actor_id).set_name(mention.full_name());
+      mention_tag
+        .set_href(actor_id.into())
+        .set_name(mention.full_name());
       tags.push(mention_tag);
     }
   }

crates/apub/src/activities/community/add_mod.rs

@@ -9,7 +9,7 @@ use crate::{
   },
   activity_queue::send_to_community_new,
   extensions::context::lemmy_context,
-  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
+  fetcher::object_id::ObjectId,
   generate_moderators_url,
   ActorType,
 };
@@ -34,11 +34,11 @@ use url::Url;
 #[derive(Clone, Debug, Deserialize, Serialize, ActivityFields)]
 #[serde(rename_all = "camelCase")]
 pub struct AddMod {
-  actor: Url,
+  actor: ObjectId<Person>,
   to: [PublicUrl; 1],
-  object: Url,
+  object: ObjectId<Person>,
   target: Url,
-  cc: [Url; 1],
+  cc: [ObjectId<Community>; 1],
   #[serde(rename = "type")]
   kind: AddType,
   id: Url,
@@ -57,11 +57,11 @@ impl AddMod {
   ) -> Result<(), LemmyError> {
     let id = generate_activity_id(AddType::Add)?;
     let add = AddMod {
-      actor: actor.actor_id(),
+      actor: ObjectId::new(actor.actor_id()),
       to: [PublicUrl::Public],
-      object: added_mod.actor_id(),
+      object: ObjectId::new(added_mod.actor_id()),
       target: generate_moderators_url(&community.actor_id)?.into(),
-      cc: [community.actor_id()],
+      cc: [ObjectId::new(community.actor_id())],
       kind: AddType::Add,
       id: id.clone(),
       context: lemmy_context(),
@@ -84,7 +84,7 @@ impl ActivityHandler for AddMod {
     verify_activity(self)?;
     verify_person_in_community(&self.actor, &self.cc[0], context, request_counter).await?;
     verify_mod_action(&self.actor, self.cc[0].clone(), context).await?;
-    verify_add_remove_moderator_target(&self.target, self.cc[0].clone())?;
+    verify_add_remove_moderator_target(&self.target, &self.cc[0])?;
     Ok(())
   }
 
@@ -93,9 +93,8 @@ impl ActivityHandler for AddMod {
     context: &LemmyContext,
     request_counter: &mut i32,
   ) -> Result<(), LemmyError> {
-    let community =
-      get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?;
-    let new_mod = get_or_fetch_and_upsert_person(&self.object, context, request_counter).await?;
+    let community = self.cc[0].dereference(context, request_counter).await?;
+    let new_mod = self.object.dereference(context, request_counter).await?;
 
     // If we had to refetch the community while parsing the activity, then the new mod has already
     // been added. Skip it here as it would result in a duplicate key error.

crates/apub/src/activities/community/announce.rs

@@ -19,6 +19,7 @@ use crate::{
   },
   activity_queue::send_activity_new,
   extensions::context::lemmy_context,
+  fetcher::object_id::ObjectId,
   http::is_activity_already_known,
   insert_activity,
   ActorType,
@@ -57,7 +58,7 @@ pub enum AnnouncableActivities {
 #[derive(Clone, Debug, Deserialize, Serialize, ActivityFields)]
 #[serde(rename_all = "camelCase")]
 pub struct AnnounceActivity {
-  actor: Url,
+  actor: ObjectId<Community>,
   to: [PublicUrl; 1],
   object: AnnouncableActivities,
   cc: Vec<Url>,
@@ -78,7 +79,7 @@ impl AnnounceActivity {
     context: &LemmyContext,
   ) -> Result<(), LemmyError> {
     let announce = AnnounceActivity {
-      actor: community.actor_id(),
+      actor: ObjectId::new(community.actor_id()),
       to: [PublicUrl::Public],
       object,
       cc: vec![community.followers_url()],

crates/apub/src/activities/community/block_user.rs

@@ -8,7 +8,7 @@ use crate::{
   },
   activity_queue::send_to_community_new,
   extensions::context::lemmy_context,
-  fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person},
+  fetcher::object_id::ObjectId,
   ActorType,
 };
 use activitystreams::{
@@ -38,10 +38,10 @@ use url::Url;
 #[derive(Clone, Debug, Deserialize, Serialize, ActivityFields)]
 #[serde(rename_all = "camelCase")]
 pub struct BlockUserFromCommunity {
-  actor: Url,
+  actor: ObjectId<Person>,
   to: [PublicUrl; 1],
-  pub(in crate::activities::community) object: Url,
-  cc: [Url; 1],
+  pub(in crate::activities::community) object: ObjectId<Person>,
+  cc: [ObjectId<Community>; 1],
   #[serde(rename = "type")]
   kind: BlockType,
   id: Url,
@@ -58,10 +58,10 @@ impl BlockUserFromCommunity {
     actor: &Person,
   ) -> Result<BlockUserFromCommunity, LemmyError> {
     Ok(BlockUserFromCommunity {
-      actor: actor.actor_id(),
+      actor: ObjectId::new(actor.actor_id()),
       to: [PublicUrl::Public],
-      object: target.actor_id(),
-      cc: [community.actor_id()],
+      object: ObjectId::new(target.actor_id()),
+      cc: [ObjectId::new(community.actor_id())],
       kind: BlockType::Block,
       id: generate_activity_id(BlockType::Block)?,
       context: lemmy_context(),
@@ -102,10 +102,8 @@ impl ActivityHandler for BlockUserFromCommunity {
     context: &LemmyContext,
     request_counter: &mut i32,
   ) -> Result<(), LemmyError> {
-    let community =
-      get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?;
-    let blocked_user =
-      get_or_fetch_and_upsert_person(&self.object, context, request_counter).await?;
+    let community = self.cc[0].dereference(context, request_counter).await?;
+    let blocked_user = self.object.dereference(context, request_counter).await?;
 
     let community_user_ban_form = CommunityPersonBanForm {
       community_id: community.id,