Extraction phases — DevRev

Each snap-in must handle all the phases of Airdrop extraction. In a snap-in, you typically define a run function that iterates over events and invokes workers per extraction phase.

The SDK library exports processTask to structure the work within each phase, and onTimeout function to handle timeouts.

The Airdrop snap-in extraction lifecycle consists of four phases:

External sync units extraction
Metadata extraction
Data extraction
Attachments extraction

Each phase is defined in a separate file and is responsible for fetching the respective data.

The SDK library provides a repository management system to handle artifacts in batches. The initializeRepos function initializes the repositories, and the push function uploads the artifacts to the repositories. The postState function is used to post the state of the extraction task.

State management is crucial for snap-ins to maintain the state of the extraction task. The postState function is used to post the state of the extraction task. The state is stored in the adapter and can be retrieved using the adapter.state property.

1 import { AirdropEvent, EventType, spawn } from "@devrev/ts-adaas";
2 
3 export interface ExtractorState {
4   todos: { completed: boolean };
5   users: { completed: boolean };
6   attachments: { completed: boolean };
7 }
8 
9 export const initialState: ExtractorState = {
10   todos: { completed: false },
11   users: { completed: false },
12   attachments: { completed: false },
13 };
14 
15 function getWorkerPerExtractionPhase(event: AirdropEvent) {
16   let path;
17   switch (event.payload.event_type) {
18     case EventType.ExtractionExternalSyncUnitsStart:
19       path = __dirname + '/workers/external-sync-units-extraction';
20       break;
21     case EventType.ExtractionMetadataStart:
22       path = __dirname + '/workers/metadata-extraction';
23       break;
24     case EventType.ExtractionDataStart:
25     case EventType.ExtractionDataContinue:
26       path = __dirname + '/workers/data-extraction';
27       break;
28     case EventType.ExtractionAttachmentsStart:
29     case EventType.ExtractionAttachmentsContinue:
30       path = __dirname + '/workers/attachments-extraction';
31       break;
32   }
33   return path;
34 }
35 
36 const run = async (events: AirdropEvent[]) => {
37   for (const event of events) {
38     const file = getWorkerPerExtractionPhase(event);
39     await spawn<ExtractorState>({
40       event,
41       initialState,
42       workerPath: file,
43       // options: {},
44     });
45   }
46 };
47 
48 export default run;

1	import { AirdropEvent, EventType, spawn } from "@devrev/ts-adaas";
2
3	export interface ExtractorState {
4	todos: { completed: boolean };
5	users: { completed: boolean };
6	attachments: { completed: boolean };
7	}
8
9	export const initialState: ExtractorState = {
10	todos: { completed: false },
11	users: { completed: false },
12	attachments: { completed: false },
13	};
14
15	function getWorkerPerExtractionPhase(event: AirdropEvent) {
16	let path;
17	switch (event.payload.event_type) {
18	case EventType.ExtractionExternalSyncUnitsStart:
19	path = __dirname + '/workers/external-sync-units-extraction';
20	break;
21	case EventType.ExtractionMetadataStart:
22	path = __dirname + '/workers/metadata-extraction';
23	break;
24	case EventType.ExtractionDataStart:
25	case EventType.ExtractionDataContinue:
26	path = __dirname + '/workers/data-extraction';
27	break;
28	case EventType.ExtractionAttachmentsStart:
29	case EventType.ExtractionAttachmentsContinue:
30	path = __dirname + '/workers/attachments-extraction';
31	break;
32	}
33	return path;
34	}
35
36	const run = async (events: AirdropEvent[]) => {
37	for (const event of events) {
38	const file = getWorkerPerExtractionPhase(event);
39	await spawn<ExtractorState>({
40	event,
41	initialState,
42	workerPath: file,
43	// options: {},
44	});
45	}
46	};
47
48	export default run;