Skip to content

Commit

Permalink
Index on search in batches
Browse files Browse the repository at this point in the history
  • Loading branch information
@handotdev
handotdev committed Jun 27, 2022
1 parent 7e22d68 commit b21fdb1
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 57 deletions.
50 changes: 27 additions & 23 deletions server/src/helpers/routes/docs.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { Types } from 'mongoose';
import Doc from '../../models/Doc';
import Doc, { DocType } from '../../models/Doc';
import Org, { OrgType } from '../../models/Org';
import { ConfluencePage } from '../../routes/integrations/confluence';
import { GoogleDoc } from '../../routes/integrations/google';
import { NotionPage } from '../../routes/integrations/notion';
import { clearIndexWithMethod, indexDocForSearch } from '../../services/algolia';
import { clearIndexWithMethod, indexDocsForSearch } from '../../services/algolia';
import { getGoogleDocsPrivateData } from '../../services/googleDocs';
import { getNotionPageDataWithId } from '../../services/notion';
import { track } from '../../services/segment';
Expand All @@ -20,7 +20,7 @@ export const importDocsFromNotion = async (pages: NotionPage[], org: OrgType, us
const orgId = org._id;
const method = 'notion-private';
await clearIndexWithMethod(orgId.toString(), method);
const addDocPromises = pages.map((page) => new Promise<void>(async (resolve) => {
const addDocPromises = pages.map((page) => new Promise<DocType | null>(async (resolve) => {
try {
if (org.integrations?.notion?.access_token == null) {
throw 'No Notion credentials'
Expand Down Expand Up @@ -58,29 +58,30 @@ export const importDocsFromNotion = async (pages: NotionPage[], org: OrgType, us
}
);

indexDocForSearch(doc);
track(userId, 'Add documentation', {
doc: doc._id.toString(),
method,
org: orgId.toString(),
});

resolve();
resolve(doc);
}
catch {
resolve();
resolve(null);
}
}));

await Promise.all(addDocPromises);
const docsResponses = await Promise.all(addDocPromises);
const docs = docsResponses.filter((doc) => doc != null) as DocType[];
indexDocsForSearch(docs);
await updateImportStatus(orgId, 'notion', false);
}

export const importDocsFromGoogleDocs = async (docs: GoogleDoc[], org: OrgType, userId: string) => {
export const importDocsFromGoogleDocs = async (googleDocs: GoogleDoc[], org: OrgType, userId: string) => {
const orgId = org._id;
const method = 'googledocs-private';
await clearIndexWithMethod(orgId.toString(), method);
const addDocPromises = docs.map((googleDoc) => new Promise<void>(async (resolve) => {
const addDocPromises = googleDocs.map((googleDoc) => new Promise<DocType | null>(async (resolve) => {
try {
if (org.integrations?.google == null) {
throw 'No Google credentials'
Expand Down Expand Up @@ -111,29 +112,30 @@ export const importDocsFromGoogleDocs = async (docs: GoogleDoc[], org: OrgType,
}
);

indexDocForSearch(doc);
track(userId, 'Add documentation', {
doc: doc._id.toString(),
method,
org: orgId.toString(),
});

resolve();
resolve(doc);
}
catch {
resolve();
resolve(null);
}
}));

await Promise.all(addDocPromises);
const docsResponses = await Promise.all(addDocPromises);
const docs = docsResponses.filter((doc) => doc != null) as DocType[];
indexDocsForSearch(docs);
await updateImportStatus(orgId, 'googledocs', false);
};

export const importDocsFromConfluence = async (pages: ConfluencePage[], org: OrgType, userId: string) => {
const orgId = org._id;
const method = 'confluence-private';
await clearIndexWithMethod(orgId.toString(), method);
const addDocPromises = pages.map((page) => new Promise<void>(async (resolve) => {
const addDocPromises = pages.map((page) => new Promise<DocType | null>(async (resolve) => {
try {
const firstSpace = org?.integrations?.confluence?.accessibleResources[0];
if (firstSpace == null) {
Expand Down Expand Up @@ -165,21 +167,22 @@ export const importDocsFromConfluence = async (pages: ConfluencePage[], org: Org
}
);

indexDocForSearch(doc);
track(userId, 'Add documentation', {
doc: doc._id.toString(),
method,
org: orgId.toString(),
});

resolve();
resolve(doc);
}
catch {
resolve();
resolve(null);
}
}));

await Promise.all(addDocPromises);
const docsResponses = await Promise.all(addDocPromises);
const docs = docsResponses.filter((doc) => doc != null) as DocType[];
indexDocsForSearch(docs);
await updateImportStatus(orgId, 'confluence', false);
};

Expand All @@ -195,7 +198,7 @@ export const importDocsFromGitHub = async (markdowns: GitHubMarkdown[], org: Org
const orgId = org._id;
const method = 'github';
await clearIndexWithMethod(orgId.toString(), method);
const addDocPromises = markdowns.map((markdown) => new Promise<void>(async (resolve) => {
const addDocPromises = markdowns.map((markdown) => new Promise<DocType | null>(async (resolve) => {
try {
const orgId = org._id;
const url = markdown.url;
Expand Down Expand Up @@ -230,20 +233,21 @@ export const importDocsFromGitHub = async (markdowns: GitHubMarkdown[], org: Org
}
);

indexDocForSearch(doc);
track(userId, 'Add documentation', {
doc: doc._id.toString(),
method,
org: orgId.toString(),
});

resolve();
resolve(doc);
}
catch {
resolve();
resolve(null);
}
}));

await Promise.all(addDocPromises);
const docsResponses = await Promise.all(addDocPromises);
const docs = docsResponses.filter((doc) => doc != null) as DocType[];
indexDocsForSearch(docs);
await updateImportStatus(orgId, 'github', false);
};
11 changes: 7 additions & 4 deletions server/src/routes/docs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { userMiddleware } from './user';
import Doc, { DocType } from '../models/Doc';
import Event from '../models/Event';
import { ContentData, ScrapingMethod } from '../services/webscraper';
import { deleteDocForSearch, indexDocForSearch } from '../services/algolia';
import { deleteDocForSearch, indexDocsForSearch } from '../services/algolia';
import Org from '../models/Org';
import { getNotionPageDataWithId } from '../services/notion';
import { getConfluenceContentFromPageById } from './integrations/confluence';
Expand Down Expand Up @@ -216,16 +216,19 @@ docsRouter.post('/webpage', userMiddleware, async (req, res) => {

try {
const { title, favicon, urlWithProtocol } = await getDataFromUrl(url);
const doc = await Doc.create({
const doc = await Doc.findOneAndUpdate({
org: org._id,
url: urlWithProtocol,
}, {
org: org._id,
url: urlWithProtocol,
method: 'web',
favicon,
title,
isJustAdded: true,
createdBy: userId
});
indexDocForSearch(doc);
}, { upsert: true, new: true });
indexDocsForSearch([doc]);
res.send({doc})
} catch (error) {
res.status(500).send({error});
Expand Down
43 changes: 13 additions & 30 deletions server/src/services/algolia.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,37 +32,20 @@ export const clearIndexWithMethod = async (orgId: string, method: string) => {
}
}

export const indexDocForSearch = async (doc: DocType) => {
export const indexDocsForSearch = async (docs: DocType[]) => {
try {
const record = {
objectID: doc._id,
name: doc.title,
content: doc.content,
url: doc.url,
org: doc.org,
favicon: doc.favicon,
method: doc.method,
};
await docsIndex.saveObject(record);
}
catch (error) {
// Todo: manage oversize
console.log(error);
}
}

export const updateDocContentForSearch = async (doc: DocType, newContent: string) => {
try {
const record = {
objectID: doc._id,
name: doc.title,
content: newContent,
url: doc.url,
org: doc.org,
favicon: doc.favicon,
method: doc.method,
};
await docsIndex.saveObject(record);
const records = docs.map((doc) => {
return {
objectID: doc._id,
name: doc.title,
content: doc.content,
url: doc.url,
org: doc.org,
favicon: doc.favicon,
method: doc.method,
};
})
await docsIndex.partialUpdateObjects(records, { createIfNotExists: true });
}
catch (error) {
// Todo: manage oversize
Expand Down

1 comment on commit b21fdb1

@vercel
Copy link

@vercel vercel bot commented on b21fdb1 Jun 27, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

connect – ./

*.mintlify.com
connect-mintlify.vercel.app
connect-git-main-mintlify.vercel.app

Please sign in to comment.