Merge branch 'entity-store-refactoring' into 'develop'

Entity store refactoring (improve performance, etc)

See merge request soapbox-pub/soapbox!2346

app/soapbox/entity-store/__tests__/reducer.test.ts

@@ -0,0 +1,79 @@
+import { entitiesFetchFail, entitiesFetchRequest, importEntities } from '../actions';
+import reducer from '../reducer';
+
+import type { EntityCache } from '../types';
+
+interface TestEntity {
+  id: string
+  msg: string
+}
+
+test('import entities', () => {
+  const entities: TestEntity[] = [
+    { id: '1', msg: 'yolo' },
+    { id: '2', msg: 'benis' },
+    { id: '3', msg: 'boop' },
+  ];
+
+  const action = importEntities(entities, 'TestEntity');
+  const result = reducer(undefined, action);
+  const cache = result.TestEntity as EntityCache<TestEntity>;
+
+  expect(cache.store['1']!.msg).toBe('yolo');
+  expect(Object.values(cache.lists).length).toBe(0);
+});
+
+test('import entities into a list', () => {
+  const entities: TestEntity[] = [
+    { id: '1', msg: 'yolo' },
+    { id: '2', msg: 'benis' },
+    { id: '3', msg: 'boop' },
+  ];
+
+  const action = importEntities(entities, 'TestEntity', 'thingies');
+  const result = reducer(undefined, action);
+  const cache = result.TestEntity as EntityCache<TestEntity>;
+
+  expect(cache.store['2']!.msg).toBe('benis');
+  expect(cache.lists.thingies?.ids.size).toBe(3);
+
+  // Now try adding an additional item.
+  const entities2: TestEntity[] = [
+    { id: '4', msg: 'hehe' },
+  ];
+
+  const action2 = importEntities(entities2, 'TestEntity', 'thingies');
+  const result2 = reducer(result, action2);
+  const cache2 = result2.TestEntity as EntityCache<TestEntity>;
+
+  expect(cache2.store['4']!.msg).toBe('hehe');
+  expect(cache2.lists.thingies?.ids.size).toBe(4);
+
+  // Finally, update an item.
+  const entities3: TestEntity[] = [
+    { id: '2', msg: 'yolofam' },
+  ];
+
+  const action3 = importEntities(entities3, 'TestEntity', 'thingies');
+  const result3 = reducer(result2, action3);
+  const cache3 = result3.TestEntity as EntityCache<TestEntity>;
+
+  expect(cache3.store['2']!.msg).toBe('yolofam');
+  expect(cache3.lists.thingies?.ids.size).toBe(4); // unchanged
+});
+
+test('fetching updates the list state', () => {
+  const action = entitiesFetchRequest('TestEntity', 'thingies');
+  const result = reducer(undefined, action);
+
+  expect(result.TestEntity!.lists.thingies!.state.fetching).toBe(true);
+});
+
+test('failure adds the error to the state', () => {
+  const error = new Error('whoopsie');
+
+  const action = entitiesFetchFail('TestEntity', 'thingies', error);
+  const result = reducer(undefined, action);
+
+  expect(result.TestEntity!.lists.thingies!.state.error).toBe(error);
+});

app/soapbox/entity-store/hooks/useEntities.ts

@@ -1,11 +1,13 @@
 import { useEffect } from 'react';
+import z from 'zod';
 
 import { getNextLink, getPrevLink } from 'soapbox/api';
-import { useApi, useAppDispatch, useAppSelector } from 'soapbox/hooks';
+import { useApi, useAppDispatch, useAppSelector, useGetState } from 'soapbox/hooks';
+import { filteredArray } from 'soapbox/schemas/utils';
 
 import { entitiesFetchFail, entitiesFetchRequest, entitiesFetchSuccess } from '../actions';
 
-import type { Entity } from '../types';
+import type { Entity, EntityListState } from '../types';
 import type { RootState } from 'soapbox/store';
 
 /** Tells us where to find/store the entity in the cache. */
@@ -17,9 +19,9 @@ type EntityPath = [
 ]
 
 /** Additional options for the hook. */
-interface UseEntitiesOpts<TEntity> {
+interface UseEntitiesOpts<TEntity extends Entity> {
-  /** A parser function that returns the desired type, or undefined if validation fails. */
+  /** A zod schema to parse the API entities. */
-  parser?: (entity: unknown) => TEntity | undefined
+  schema?: z.ZodType<TEntity, z.ZodTypeDef, any>
   /**
    * Time (milliseconds) until this query becomes stale and should be refetched.
    * It is 1 minute by default, and can be set to `Infinity` to opt-out of automatic fetching.
@@ -38,41 +40,29 @@ function useEntities<TEntity extends Entity>(
 ) {
   const api = useApi();
   const dispatch = useAppDispatch();
+  const getState = useGetState();
 
   const [entityType, listKey] = path;
+  const entities = useAppSelector(state => selectEntities<TEntity>(state, path));
 
-  const defaultParser = (entity: unknown) => entity as TEntity;
+  const isFetching = useListState(path, 'fetching');
-  const parseEntity = opts.parser || defaultParser;
+  const lastFetchedAt = useListState(path, 'lastFetchedAt');
 
-  const cache = useAppSelector(state => state.entities[entityType]);
+  const next = useListState(path, 'next');
-  const list = cache?.lists[listKey];
+  const prev = useListState(path, 'prev');
-
-  const entityIds = list?.ids;
-
-  const entities: readonly TEntity[] = entityIds ? (
-    Array.from(entityIds).reduce<TEntity[]>((result, id) => {
-      const entity = parseEntity(cache?.store[id] as unknown);
-      if (entity) {
-        result.push(entity);
-      }
-      return result;
-    }, [])
-  ) : [];
-
-  const isFetching = Boolean(list?.state.fetching);
-  const isLoading = isFetching && entities.length === 0;
-  const hasNextPage = Boolean(list?.state.next);
-  const hasPreviousPage = Boolean(list?.state.prev);
 
   const fetchPage = async(url: string): Promise<void> => {
     // Get `isFetching` state from the store again to prevent race conditions.
-    const isFetching = dispatch((_, getState: () => RootState) => Boolean(getState().entities[entityType]?.lists[listKey]?.state.fetching));
+    const isFetching = selectListState(getState(), path, 'fetching');
     if (isFetching) return;
 
     dispatch(entitiesFetchRequest(entityType, listKey));
     try {
       const response = await api.get(url);
-      dispatch(entitiesFetchSuccess(response.data, entityType, listKey, {
+      const schema = opts.schema || z.custom<TEntity>();
+      const entities = filteredArray(schema).parse(response.data);
+
+      dispatch(entitiesFetchSuccess(entities, entityType, listKey, {
         next: getNextLink(response),
         prev: getPrevLink(response),
         fetching: false,
@@ -91,23 +81,18 @@ function useEntities<TEntity extends Entity>(
   };
 
   const fetchNextPage = async(): Promise<void> => {
-    const next = list?.state.next;
-
     if (next) {
       await fetchPage(next);
     }
   };
 
   const fetchPreviousPage = async(): Promise<void> => {
-    const prev = list?.state.prev;
-
     if (prev) {
       await fetchPage(prev);
     }
   };
 
   const staleTime = opts.staleTime ?? 60000;
-  const lastFetchedAt = list?.state.lastFetchedAt;
 
   useEffect(() => {
     if (!isFetching && (!lastFetchedAt || lastFetchedAt.getTime() + staleTime <= Date.now())) {
@@ -119,14 +104,49 @@ function useEntities<TEntity extends Entity>(
     entities,
     fetchEntities,
     isFetching,
-    isLoading,
+    isLoading: isFetching && entities.length === 0,
-    hasNextPage,
+    hasNextPage: !!next,
-    hasPreviousPage,
+    hasPreviousPage: !!prev,
     fetchNextPage,
     fetchPreviousPage,
   };
 }
 
+/** Get cache at path from Redux. */
+const selectCache = (state: RootState, path: EntityPath) => state.entities[path[0]];
+
+/** Get list at path from Redux. */
+const selectList = (state: RootState, path: EntityPath) => selectCache(state, path)?.lists[path[1]];
+
+/** Select a particular item from a list state. */
+function selectListState<K extends keyof EntityListState>(state: RootState, path: EntityPath, key: K) {
+  const listState = selectList(state, path)?.state;
+  return listState ? listState[key] : undefined;
+}
+
+/** Hook to get a particular item from a list state. */
+function useListState<K extends keyof EntityListState>(path: EntityPath, key: K) {
+  return useAppSelector(state => selectListState(state, path, key));
+}
+
+/** Get list of entities from Redux. */
+function selectEntities<TEntity extends Entity>(state: RootState, path: EntityPath): readonly TEntity[] {
+  const cache = selectCache(state, path);
+  const list = selectList(state, path);
+
+  const entityIds = list?.ids;
+
+  return entityIds ? (
+    Array.from(entityIds).reduce<TEntity[]>((result, id) => {
+      const entity = cache?.store[id];
+      if (entity) {
+        result.push(entity as TEntity);
+      }
+      return result;
+    }, [])
+  ) : [];
+}
+
 export {
   useEntities,
 };

app/soapbox/entity-store/hooks/useEntity.ts

@@ -1,4 +1,5 @@
 import { useEffect, useState } from 'react';
+import z from 'zod';
 
 import { useApi, useAppDispatch, useAppSelector } from 'soapbox/hooks';
 
@@ -10,8 +11,8 @@ type EntityPath = [entityType: string, entityId: string]
 
 /** Additional options for the hook. */
 interface UseEntityOpts<TEntity> {
-  /** A parser function that returns the desired type, or undefined if validation fails. */
+  /** A zod schema to parse the API entity. */
-  parser?: (entity: unknown) => TEntity | undefined
+  schema?: z.ZodType<TEntity, z.ZodTypeDef, any>
   /** Whether to refetch this entity every time the hook mounts, even if it's already in the store. */
   refetch?: boolean
 }
@@ -26,10 +27,10 @@ function useEntity<TEntity extends Entity>(
 
   const [entityType, entityId] = path;
 
-  const defaultParser = (entity: unknown) => entity as TEntity;
+  const defaultSchema = z.custom<TEntity>();
-  const parseEntity = opts.parser || defaultParser;
+  const schema = opts.schema || defaultSchema;
 
-  const entity = useAppSelector(state => parseEntity(state.entities[entityType]?.store[entityId]));
+  const entity = useAppSelector(state => state.entities[entityType]?.store[entityId] as TEntity | undefined);
 
   const [isFetching, setIsFetching] = useState(false);
   const isLoading = isFetching && !entity;
@@ -37,7 +38,8 @@ function useEntity<TEntity extends Entity>(
   const fetchEntity = () => {
     setIsFetching(true);
     api.get(endpoint).then(({ data }) => {
-      dispatch(importEntities([data], entityType));
+      const entity = schema.parse(data);
+      dispatch(importEntities([entity], entityType));
       setIsFetching(false);
     }).catch(() => {
       setIsFetching(false);

app/soapbox/entity-store/reducer.ts

@@ -48,6 +48,7 @@ const setFetching = (
   entityType: string,
   listKey: string | undefined,
   isFetching: boolean,
+  error?: any,
 ) => {
   return produce(state, draft => {
     const cache = draft[entityType] ?? createCache();
@@ -55,6 +56,7 @@ const setFetching = (
     if (typeof listKey === 'string') {
       const list = cache.lists[listKey] ?? createList();
       list.state.fetching = isFetching;
+      list.state.error = error;
       cache.lists[listKey] = list;
     }
 
@@ -72,7 +74,7 @@ function reducer(state: Readonly<State> = {}, action: EntityAction): State {
     case ENTITIES_FETCH_REQUEST:
       return setFetching(state, action.entityType, action.listKey, true);
     case ENTITIES_FETCH_FAIL:
-      return setFetching(state, action.entityType, action.listKey, false);
+      return setFetching(state, action.entityType, action.listKey, false, action.error);
     default:
       return state;
   }

app/soapbox/entity-store/types.ts

@@ -5,8 +5,8 @@ interface Entity {
 }
 
 /** Store of entities by ID. */
-interface EntityStore {
+interface EntityStore<TEntity extends Entity = Entity> {
-  [id: string]: Entity | undefined
+  [id: string]: TEntity | undefined
 }
 
 /** List of entity IDs and fetch state. */
@@ -32,9 +32,9 @@ interface EntityListState {
 }
 
 /** Cache data pertaining to a paritcular entity type.. */
-interface EntityCache {
+interface EntityCache<TEntity extends Entity = Entity> {
   /** Map of entities of this type. */
-  store: EntityStore
+  store: EntityStore<TEntity>
   /** Lists of entity IDs for a particular purpose. */
   lists: {
     [listKey: string]: EntityList | undefined

app/soapbox/hooks/index.ts

@@ -5,6 +5,7 @@ export { useAppSelector } from './useAppSelector';
 export { useClickOutside } from './useClickOutside';
 export { useCompose } from './useCompose';
 export { useDebounce } from './useDebounce';
+export { useGetState } from './useGetState';
 export { useGroup, useGroups } from './useGroups';
 export { useGroupsPath } from './useGroupsPath';
 export { useDimensions } from './useDimensions';

app/soapbox/hooks/useApi.ts

@@ -1,12 +1,9 @@
 import api from 'soapbox/api';
 
-import { useAppDispatch } from './useAppDispatch';
+import { useGetState } from './useGetState';
 
 /** Use stateful Axios client with auth from Redux. */
 export const useApi = () => {
-  const dispatch = useAppDispatch();
+  const getState = useGetState();
-
+  return api(getState);
-  return dispatch((_dispatch, getState) => {
-    return api(getState);
-  });
 };

app/soapbox/hooks/useGetState.ts

@@ -0,0 +1,14 @@
+import { useAppDispatch } from './useAppDispatch';
+
+import type { RootState } from 'soapbox/store';
+
+/**
+ * Provides a `getState()` function to hooks.
+ * You should prefer `useAppSelector` when possible.
+ */
+function useGetState() {
+  const dispatch = useAppDispatch();
+  return () => dispatch((_, getState: () => RootState) => getState());
+}
+
+export { useGetState };

app/soapbox/hooks/useGroups.ts

@@ -3,7 +3,7 @@ import { groupSchema, Group } from 'soapbox/schemas/group';
 import { groupRelationshipSchema, GroupRelationship } from 'soapbox/schemas/group-relationship';
 
 function useGroups() {
-  const { entities, ...result } = useEntities<Group>(['Group', ''], '/api/v1/groups', { parser: parseGroup });
+  const { entities, ...result } = useEntities<Group>(['Group', ''], '/api/v1/groups', { schema: groupSchema });
   const { relationships } = useGroupRelationships(entities.map(entity => entity.id));
 
   const groups = entities.map((group) => ({ ...group, relationship: relationships[group.id] || null }));
@@ -15,7 +15,7 @@ function useGroups() {
 }
 
 function useGroup(groupId: string, refetch = true) {
-  const { entity: group, ...result } = useEntity<Group>(['Group', groupId], `/api/v1/groups/${groupId}`, { parser: parseGroup, refetch });
+  const { entity: group, ...result } = useEntity<Group>(['Group', groupId], `/api/v1/groups/${groupId}`, { schema: groupSchema, refetch });
   const { entity: relationship } = useGroupRelationship(groupId);
 
   return {
@@ -25,13 +25,13 @@ function useGroup(groupId: string, refetch = true) {
 }
 
 function useGroupRelationship(groupId: string) {
-  return useEntity<GroupRelationship>(['GroupRelationship', groupId], `/api/v1/groups/relationships?id[]=${groupId}`, { parser: parseGroupRelationship });
+  return useEntity<GroupRelationship>(['GroupRelationship', groupId], `/api/v1/groups/relationships?id[]=${groupId}`, { schema: groupRelationshipSchema });
 }
 
 function useGroupRelationships(groupIds: string[]) {
   const q = groupIds.map(id => `id[]=${id}`).join('&');
   const endpoint = groupIds.length ? `/api/v1/groups/relationships?${q}` : undefined;
-  const { entities, ...result } = useEntities<GroupRelationship>(['GroupRelationship', q], endpoint, { parser: parseGroupRelationship });
+  const { entities, ...result } = useEntities<GroupRelationship>(['GroupRelationship', q], endpoint, { schema: groupRelationshipSchema });
 
   const relationships = entities.reduce<Record<string, GroupRelationship>>((map, relationship) => {
     map[relationship.id] = relationship;
@@ -44,18 +44,4 @@ function useGroupRelationships(groupIds: string[]) {
   };
 }
 
-const parseGroup = (entity: unknown) => {
-  const result = groupSchema.safeParse(entity);
-  if (result.success) {
-    return result.data;
-  }
-};
-
-const parseGroupRelationship = (entity: unknown) => {
-  const result = groupRelationshipSchema.safeParse(entity);
-  if (result.success) {
-    return result.data;
-  }
-};
-
 export { useGroup, useGroups };

app/soapbox/schemas/utils.ts

@@ -4,10 +4,13 @@ import type { CustomEmoji } from './custom-emoji';
 
 /** Validates individual items in an array, dropping any that aren't valid. */
 function filteredArray<T extends z.ZodTypeAny>(schema: T) {
-  return z.any().array().transform((arr) => (
+  return z.any().array()
-    arr.map((item) => schema.safeParse(item).success ? item as z.infer<T> : undefined)
+    .transform((arr) => (
-      .filter((item): item is z.infer<T> => Boolean(item))
+      arr.map((item) => {
-  ));
+        const parsed = schema.safeParse(item);
+        return parsed.success ? parsed.data : undefined;
+      }).filter((item): item is z.infer<T> => Boolean(item))
+    ));
 }
 
 /** Map a list of CustomEmoji to their shortcodes. */