Add loadingTime/decodingTime into dataset timings, changes in stages

CHANGELOG.md

@@ -9,13 +9,19 @@ export function async prepare(input, { setWorkTitle: (title: string) => Promise<
  // ...
 }
 ```
-- Refactor `Progressbar`
+- Refactor `Progressbar`:
  - Added `setStateStep(step)` method to set a secondary text for the stage
  - Changed `setState()` method to take second optional parameter `step`
  - Modified logic for await repainting
  - Added `awaitRepaintPenaltyTime` property to indicate time spending on awaiting for repaint
  - Changed `onFinish` callback to add `awaitRepaintPenaltyTime` to `timings` array
+ - Added `decoding` stage
+ - Renamed `receive` stage into `receiving`
  - Removed `lastStage` as it redundant, use `value.stage` instead
+- Changes in data loading:
+ - Added `decoding` stage for load state
+ - Renamed `receive` stage into `receiving` for load state
+ - Added `loadingTime` and `decodingTime` into dataset timings
 - Fixed crashing the entire render tree on an exception in a view's `render` function; now, crashes are isolated to the affected view
 - Fixed unnecessary view rendering when returning to the discovery page
 - Fixed hiding a popup with `hideOnResize: true` when scrolling outside of the popup element

src/core/encodings/json.ts

@@ -6,4 +6,4 @@ export default Object.freeze({
  test: () => true,
  streaming: true,
  decode: parseChunked
-}) satisfies Encoding;
+}) satisfies Encoding as Encoding;

src/core/encodings/jsonxl.ts

@@ -6,4 +6,4 @@ export default Object.freeze({
  test,
  streaming: false,
  decode
-}) satisfies Encoding;
+}) satisfies Encoding as Encoding;

src/core/utils/load-data.ts

@@ -139,7 +139,11 @@ export async function dataFromStream(
  stream: ReadableStream<Uint8Array>,
  extraEncodings: Encoding[] | undefined,
  totalSize: number | undefined,
- setProgress: (state: LoadDataStateProgress) => Promise<boolean>
+ setStageProgress: (
+ stage: 'receiving' | 'decoding',
+ progress?: LoadDataStateProgress,
+ step?: string
+ ) => Promise<boolean>
 ) {
  const CHUNK_SIZE = 1024 * 1024; // 1MB
  const reader = stream.getReader();
@@ -149,44 +153,11 @@ export async function dataFromStream(
  buildinEncodings.jsonxl,
  buildinEncodings.json
  ];
+ let decodingTime = 0;
  let encoding = 'unknown';
  let size = 0;
 
- const streamConsumer = async function*(firstChunk?: ReadableStreamReadResult<Uint8Array>) {
- while (true) {
- const { value, done } = firstChunk || await reader.read();
-
- firstChunk = undefined;
-
- if (done) {
- await setProgress({
- done: true,
- elapsed: Date.now() - streamStartTime,
- units: 'bytes',
- completed: size,
- total: totalSize
- });
- break;
- }
-
- for (let offset = 0; offset < value.length; offset += CHUNK_SIZE) {
- const chunk = offset === 0 && value.length - offset < CHUNK_SIZE
- ? value
- : value.slice(offset, offset + CHUNK_SIZE);
-
- size += chunk.length;
- yield chunk;
-
- await setProgress({
- done: false,
- elapsed: Date.now() - streamStartTime,
- units: 'bytes',
- completed: size,
- total: totalSize
- });
- }
- }
- };
+ await setStageProgress('receiving', getProgress(false));
 
  try {
  const firstChunk = await reader.read();
@@ -202,62 +173,120 @@ export async function dataFromStream(
 
  const decodeRequest = streaming
  ? decode(streamConsumer(firstChunk))
- : consumeChunksAsSingleTypedArray(streamConsumer(firstChunk)).then(decode);
+ : consumeChunksAsSingleTypedArray(streamConsumer(firstChunk)).then(measureDecodingTime(decode));
  const data = await decodeRequest;
 
- return { data, encoding, size };
+ return { data, encoding, size, decodingTime };
  }
  }
 
  throw new Error('No matched encoding found for the payload');
  } finally {
  reader.releaseLock();
  }
+
+ function getProgress(done: boolean): LoadDataStateProgress {
+ return {
+ done,
+ elapsed: Date.now() - streamStartTime,
+ units: 'bytes',
+ completed: size,
+ total: totalSize
+ };
+ }
+
+ async function* streamConsumer(firstChunk?: ReadableStreamReadResult<Uint8Array>) {
+ while (true) {
+ const { value, done } = firstChunk || await reader.read();
+
+ firstChunk = undefined;
+
+ if (done) {
+ break;
+ }
+
+ for (let offset = 0; offset < value.length; offset += CHUNK_SIZE) {
+ const chunkDecodingStartTime = performance.now();
+ const chunk = offset === 0 && value.length - offset < CHUNK_SIZE
+ ? value
+ : value.slice(offset, offset + CHUNK_SIZE);
+
+ yield chunk;
+
+ decodingTime += performance.now() - chunkDecodingStartTime;
+ size += chunk.length;
+
+ await setStageProgress('receiving', getProgress(false));
+ }
+ }
+
+ // progress done
+ await setStageProgress('receiving', getProgress(true));
+ }
+
+ function measureDecodingTime(decode: (payload: Uint8Array) => any) {
+ return async (payload: Uint8Array) => {
+ await setStageProgress('decoding', undefined, encoding);
+
+ const startDecodingTime = performance.now();
+
+ try {
+ return await decode(payload);
+ } finally {
+ decodingTime = performance.now() - startDecodingTime;
+ }
+ };
+ }
 }
 
 async function loadDataFromStreamInternal(
  request: LoadDataRequest,
- progress: Observer<LoadDataState>
+ loadDataStateTracker: Observer<LoadDataState>
 ): Promise<Dataset> {
- const stage = async <T>(stage: 'request' | 'receive', fn: () => T): Promise<T> => {
- await progress.asyncSet({ stage });
- return await fn();
- };
-
  try {
+ await loadDataStateTracker.asyncSet({ stage: 'request' });
  const requestStart = new Date();
  const {
  method,
  stream,
  resource: rawResource,
  options,
  data: explicitData
- } = await stage('request', request);
+ } = await request();
 
  const responseStart = new Date();
  const payloadSize = rawResource?.size;
  const { encodings } = options || {};
- const { data: rawData, encoding = 'unknown', size = undefined } = explicitData ? { data: explicitData } : await stage('receive', () =>
- dataFromStream(stream, encodings, Number(payloadSize) || 0, state => progress.asyncSet({
- stage: 'receive',
- progress: state
- }))
+ const {
+ data: rawData,
+ encoding = 'unknown',
+ size = undefined,
+ decodingTime = 0
+ } = explicitData ? { data: explicitData } : await dataFromStream(
+ stream,
+ encodings,
+ Number(payloadSize) || 0,
+ (stage, progress, step) => loadDataStateTracker.asyncSet({ stage, progress, step })
  );
 
- await progress.asyncSet({ stage: 'received' });
+ await loadDataStateTracker.asyncSet({ stage: 'received' });
 
  const { data, resource, meta } = buildDataset(rawData, rawResource, { size, encoding });
  const finishedTime = new Date();
+ const time = Number(finishedTime) - Number(requestStart);
+ const roundedDecodingTime = Math.round(decodingTime || 0);
 
  return {
  loadMethod: method,
  resource,
  meta,
  data,
  timings: {
- time: Number(finishedTime) - Number(requestStart),
+ time,
  start: requestStart,
  end: finishedTime,
+ loadingTime: time - roundedDecodingTime,
+ decodingTime: roundedDecodingTime,
  requestTime: Number(responseStart) - Number(requestStart),
  requestStart,
  requestEnd: responseStart,
@@ -268,7 +297,7 @@ async function loadDataFromStreamInternal(
  };
  } catch (error) {
  console.error('[Discovery] Error loading data:', error);
- await progress.asyncSet({ stage: 'error', error });
+ await loadDataStateTracker.asyncSet({ stage: 'error', error });
  throw error;
  }
 }
@@ -412,21 +441,21 @@ export function loadDataFromPush(options?: LoadDataBaseOptions) {
 
 export function syncLoaderWithProgressbar({ dataset, state }: LoadDataResult, progressbar: Progressbar) {
  return new Promise<Dataset>((resolve, reject) =>
- state.subscribeSync((loadDataState, unsubscribe) => {
- const { stage } = loadDataState;
-
- if (stage === 'error') {
+ state.subscribeSync(async (loadDataState, unsubscribe) => {
+ if (loadDataState.stage === 'error') {
  unsubscribe();
  reject(loadDataState.error);
  return;
  }
 
+ const { stage, progress, step } = loadDataState;
+
+ await progressbar.setState({ stage, progress }, step);
+
  if (stage === 'received') {
  unsubscribe();
  resolve(dataset);
  }
-
- return progressbar.setState({ stage, progress: loadDataState.progress });
  })
  );
 }

src/core/utils/load-data.types.ts

@@ -19,12 +19,13 @@ export type LoadDataResult = {
 }
 export type LoadDataState =
  | {
- stage: 'inited' | 'request' | 'receive' | 'received',
- progress?: LoadDataStateProgress
+ stage: 'inited' | 'request' | 'receiving' | 'decoding' | 'received';
+ progress?: LoadDataStateProgress;
+ step?: string;
  }
  | {
- stage: 'error',
- error: Error
+ stage: 'error';
+ error: Error;
  };
 export type LoadDataStateProgress = {
  done: boolean;
@@ -87,6 +88,8 @@ export type DatasetTimings = {
  time: number;
  start: Date;
  end: Date;
+ loadingTime: number;
+ decodingTime: number;
  requestTime: number;
  requestStart: Date;
  requestEnd: Date;

src/core/utils/progressbar.ts

@@ -38,24 +38,29 @@ export const loadStages = {
  duration: 0.1,
  title: 'Awaiting data'
  },
- receive: {
+ receiving: {
  value: 0.1,
  duration: 0.8,
  title: 'Receiving data'
  },
- received: {
+ decoding: {
  value: 0.9,
- duration: 0.025,
+ duration: 0.015,
+ title: 'Decoding data'
+ },
+ received: {
+ value: 0.915,
+ duration: 0.01,
  title: 'Await app ready'
  },
  prepare: {
  value: 0.925,
- duration: 0.050,
+ duration: 0.055,
  title: 'Processing data (prepare)'
  },
  initui: {
- value: 0.975,
- duration: 0.025,
+ value: 0.98,
+ duration: 0.02,
  title: 'Rendering UI'
  },
  done: {
@@ -69,9 +74,6 @@ export const loadStages = {
  title: 'Error'
  }
 };
-Object.values(loadStages).forEach((item, idx, array) => {
- item.duration = (idx !== array.length - 1 ? array[idx + 1].value : 0) - item.value;
-});
 
 const int = (value: number) => value | 0;
 const ensureFunction = (value: any): (() => void) => typeof value === 'function' ? value : () => undefined;
@@ -112,6 +114,8 @@ export function decodeStageProgress(stage: Stage, progress: ProgressbarState['pr
  }
  }
 
+ console.log(stage, progressValue, value + progressValue * duration);
+
  return {
  stageTitle,
  progressValue: value + progressValue * duration,