enhance smoosh to cleanup search indexes when ddocs change

Author: rnewson

src/dreyfus/src/dreyfus_fabric_cleanup.erl

@@ -14,40 +14,17 @@
 
 -module(dreyfus_fabric_cleanup).
 
--include("dreyfus.hrl").
 -include_lib("mem3/include/mem3.hrl").
 -include_lib("couch/include/couch_db.hrl").
 
 -export([go/1]).
 
 go(DbName) ->
-    {ok, DesignDocs} = fabric:design_docs(DbName),
-    ActiveSigs = lists:usort(
-        lists:flatmap(
-            fun active_sigs/1,
-            [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
-        )
-    ),
+    ActiveSigs = dreyfus_util:active_sigs(DbName),
     cleanup_local_purge_doc(DbName, ActiveSigs),
     clouseau_rpc:cleanup(DbName, ActiveSigs),
     ok.
 
-active_sigs(#doc{body = {Fields}} = Doc) ->
-    try
-        {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}),
-        {IndexNames, _} = lists:unzip(RawIndexes),
-        [
-            begin
-                {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName),
-                Index#index.sig
-            end
-         || IndexName <- IndexNames
-        ]
-    catch
-        error:{badmatch, _Error} ->
-            []
-    end.
-
 cleanup_local_purge_doc(DbName, ActiveSigs) ->
     {ok, BaseDir} = clouseau_rpc:get_root_dir(),
     DbNamePattern = <<DbName/binary, ".*">>,

src/dreyfus/src/dreyfus_util.erl

@@ -29,7 +29,8 @@
     maybe_create_local_purge_doc/2,
     maybe_create_local_purge_doc/3,
     get_signature_from_idxdir/1,
-    verify_index_exists/2
+    verify_index_exists/2,
+    active_sigs/1
 ]).
 
 get_shards(DbName, #index_query_args{partition = nil} = Args) ->
@@ -428,6 +429,33 @@ verify_index_exists(DbName, Props) ->
             false
     end.
 
+active_sigs(DbName) when is_binary(DbName) ->
+    couch_util:with_db(DbName, fun active_sigs/1);
+active_sigs(Db) ->
+    {ok, DesignDocs} = couch_db:get_design_docs(Db),
+    lists:usort(
+        lists:flatmap(
+            fun sigs_from_ddoc/1,
+            [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
+        )
+    ).
+
+sigs_from_ddoc(#doc{body = {Fields}} = Doc) ->
+    try
+        {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}),
+        {IndexNames, _} = lists:unzip(RawIndexes),
+        [
+            begin
+                {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName),
+                Index#index.sig
+            end
+         || IndexName <- IndexNames
+        ]
+    catch
+        error:{badmatch, _Error} ->
+            []
+    end.
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 

src/fabric/src/fabric.erl

@@ -590,25 +590,38 @@ cleanup_index_files(DbName) ->
 cleanup_local_indices_and_purge_checkpoints([]) ->
     ok;
 cleanup_local_indices_and_purge_checkpoints([_ | _] = Dbs) ->
-    AllIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs),
-    AllPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs),
-    Sigs = couch_mrview_util:get_signatures(hd(Dbs)),
-    ok = cleanup_purges(Sigs, AllPurges, Dbs),
-    ok = cleanup_indices(Sigs, AllIndices).
+    MrViewIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs),
+    MrViewPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs),
+    MrViewSigs = couch_mrview_util:get_signatures(hd(Dbs)),
+    ok = cleanup_mrview_purges(MrViewSigs, MrViewPurges, Dbs),
+    ok = cleanup_mrview_indices(MrViewSigs, MrViewIndices),
 
-cleanup_purges(Sigs, AllPurges, Dbs) ->
+    ClouseauSigs = dreyfus_util:active_sigs(hd(Dbs)),
+    ok = cleanup_clouseau_indices(Dbs, ClouseauSigs),
+
+    NouveauSigs = nouveau_util:active_sigs(hd(Dbs)),
+    ok = cleanup_nouveau_indices(Dbs, NouveauSigs).
+
+cleanup_mrview_purges(Sigs, AllPurges, Dbs) ->
     Fun = fun(DbPurges, Db) ->
         couch_mrview_cleanup:cleanup_purges(Db, Sigs, DbPurges)
     end,
     lists:zipwith(Fun, AllPurges, Dbs),
     ok.
 
-cleanup_indices(Sigs, AllIndices) ->
+cleanup_mrview_indices(Sigs, AllIndices) ->
     Fun = fun(DbIndices) ->
         couch_mrview_cleanup:cleanup_indices(Sigs, DbIndices)
     end,
     lists:foreach(Fun, AllIndices).
 
+cleanup_clouseau_indices(Dbs, ActiveSigs) ->
+    Fun = fun(Db) -> clouseau_rpc:cleanup(Db, ActiveSigs) end,
+    lists:foreach(Fun, Dbs).
+cleanup_nouveau_indices(Dbs, ActiveSigs) ->
+    Fun = fun(Db) -> nouveau_api:delete_path(nouveau_util:index_name(Db), ActiveSigs) end,
+    lists:foreach(Fun, Dbs).
+
 %% @doc clean up index files for a specific db on all nodes
 -spec cleanup_index_files_all_nodes(dbname()) -> [reference()].
 cleanup_index_files_all_nodes(DbName) ->

src/nouveau/src/nouveau_fabric_cleanup.erl

@@ -14,9 +14,6 @@
 
 -module(nouveau_fabric_cleanup).
 
--include_lib("couch/include/couch_db.hrl").
-
--include("nouveau.hrl").
 -include_lib("mem3/include/mem3.hrl").
 
 -export([go/1]).
@@ -26,7 +23,7 @@ go(DbName) ->
     ActiveSigs =
         lists:usort(
             lists:flatmap(
-                fun(Doc) -> active_sigs(DbName, Doc) end,
+                fun(Doc) -> nouveau_util:active_sigs(DbName, Doc) end,
                 [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
             )
         ),
@@ -37,7 +34,3 @@ go(DbName) ->
         end,
         Shards
     ).
-
-active_sigs(DbName, #doc{} = Doc) ->
-    Indexes = nouveau_util:design_doc_to_indexes(DbName, Doc),
-    lists:map(fun(Index) -> Index#index.sig end, Indexes).

src/nouveau/src/nouveau_util.erl

@@ -22,6 +22,8 @@
     index_name/1,
     design_doc_to_indexes/2,
     design_doc_to_index/3,
+    active_sigs/1,
+    active_sigs/2,
     nouveau_url/0
 ]).
 
@@ -93,5 +95,21 @@ design_doc_to_index(DbName, #doc{id = Id, body = {Fields}}, IndexName) ->
             {error, InvalidDDocError}
     end.
 
+active_sigs(DbName) when is_binary(DbName) ->
+    couch_util:with_db(DbName, fun active_sigs/1);
+active_sigs(Db) ->
+    DbName = couch_db:name(Db),
+    {ok, DesignDocs} = couch_db:get_design_docs(Db),
+    lists:usort(
+        lists:flatmap(
+            fun(Doc) -> active_sigs(DbName, Doc) end,
+            [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
+        )
+    ).
+
+active_sigs(DbName, #doc{} = Doc) ->
+    Indexes = design_doc_to_indexes(DbName, Doc),
+    lists:map(fun(Index) -> Index#index.sig end, Indexes).
+
 nouveau_url() ->
     config:get("nouveau", "url", "http://127.0.0.1:8080").

src/smoosh/src/smoosh_server.erl

@@ -123,6 +123,9 @@ handle_db_event(DbName, local_updated, St) ->
 handle_db_event(DbName, updated, St) ->
     enqueue(DbName),
     {ok, St};
+handle_db_event(DbName, ddoc_updated, St) ->
+    enqueue({?INDEX_CLEANUP, mem3:dbname(DbName)}),
+    {ok, St};
 handle_db_event(DbName, {index_commit, IdxName}, St) ->
     enqueue({DbName, IdxName}),
     {ok, St};
@@ -342,6 +345,8 @@ find_channel(#state{} = State, [Channel | Rest], Object) ->
             find_channel(State, Rest, Object)
     end.
 
+stale_enough({?INDEX_CLEANUP, _}) ->
+    true;
 stale_enough(Object) ->
     LastUpdatedSec = last_updated(Object),
     Staleness = erlang:monotonic_time(second) - LastUpdatedSec,