This project has retired. For details please refer to its Attic page.
RepositoryFactory xref
View Javadoc
1   package org.apache.archiva.metadata.repository.jcr;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   * http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import com.google.common.collect.ImmutableSet;
23  import org.apache.commons.lang.time.StopWatch;
24  import org.apache.jackrabbit.JcrConstants;
25  import org.apache.jackrabbit.oak.Oak;
26  import org.apache.jackrabbit.oak.api.Type;
27  import org.apache.jackrabbit.oak.jcr.Jcr;
28  import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
29  import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
30  import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
31  import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
32  import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
33  import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
34  import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
35  import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver;
36  import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory;
37  import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
38  import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
39  import org.apache.jackrabbit.oak.segment.file.FileStore;
40  import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
41  import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
42  import org.apache.jackrabbit.oak.spi.commit.Observer;
43  import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
44  import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
45  import org.apache.jackrabbit.oak.spi.mount.Mounts;
46  import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
47  import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
48  import org.apache.jackrabbit.oak.spi.state.NodeStore;
49  import org.apache.jackrabbit.oak.stats.StatisticsProvider;
50  import org.slf4j.Logger;
51  import org.slf4j.LoggerFactory;
52  
53  import javax.annotation.Nonnull;
54  import javax.jcr.Repository;
55  import java.io.IOException;
56  import java.nio.file.Files;
57  import java.nio.file.Path;
58  import java.nio.file.Paths;
59  import java.util.concurrent.ExecutorService;
60  import java.util.concurrent.Executors;
61  
62  import static org.apache.archiva.metadata.repository.jcr.RepositoryFactory.StoreType.IN_MEMORY_TYPE;
63  import static org.apache.archiva.metadata.repository.jcr.RepositoryFactory.StoreType.SEGMENT_FILE_TYPE;
64  import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
65  
66  /**
67   * Created by martin on 14.06.17.
68   *
69   * @author Martin Stockhammer
70   * @since 3.0.0
71   */
72  public class RepositoryFactory
73  {
74  
75      private Logger log = LoggerFactory.getLogger( RepositoryFactory.class );
76  
77      private FileStore fileStore;
78  
79      private NodeStore nodeStore;
80  
81      private ExecutorService executorService;
82  
83      public enum StoreType
84      {
85          SEGMENT_FILE_TYPE,
86          IN_MEMORY_TYPE;
87      }
88  
89      private StoreType storeType = SEGMENT_FILE_TYPE;
90  
91      private Path repositoryPath = Paths.get( "repository" );
92  
93      public Repository createRepository()
94          throws IOException, InvalidFileStoreVersionException
95      {
96          createExecutor();
97  
98          if ( SEGMENT_FILE_TYPE == storeType )
99          {
100             fileStore = FileStoreBuilder.fileStoreBuilder( repositoryPath.toFile() ).build();
101             nodeStore = SegmentNodeStoreBuilders.builder( fileStore ) //
102                 .withStatisticsProvider( StatisticsProvider.NOOP ) //
103                 .build();
104         }
105         else if ( IN_MEMORY_TYPE == storeType )
106         {
107             nodeStore = null;
108         }
109         else
110         {
111             throw new IllegalArgumentException( "Store type " + storeType + " not recognized" );
112         }
113 
114         Oak oak = nodeStore == null ? new Oak() : new Oak( nodeStore );
115         oak.with( new RepositoryInitializer()
116         {
117             @Override
118             public void initialize( @Nonnull NodeBuilder root )
119             {
120                 log.info( "Creating index " );
121 
122                 NodeBuilder lucene = IndexUtils.getOrCreateOakIndex( root ).child( "lucene" );
123                 lucene.setProperty( JcrConstants.JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", Type.NAME );
124 
125                 lucene.setProperty( "compatVersion", 2 );
126                 lucene.setProperty( "type", "lucene" );
127                 // lucene.setProperty("async", "async");
128                 lucene.setProperty( INCLUDE_PROPERTY_TYPES, ImmutableSet.of( "String" ), Type.STRINGS );
129                 // lucene.setProperty("refresh",true);
130                 lucene.setProperty( "async", ImmutableSet.of( "async", "sync" ), Type.STRINGS );
131                 NodeBuilder rules = lucene.child( "indexRules" ).
132                     setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
133                 rules.setProperty( ":childOrder", ImmutableSet.of( "archiva:projectVersion", //
134                                                                    "archiva:artifact", //
135                                                                    "archiva:facet", //
136                                                                    "archiva:namespace", //
137                                                                    "archiva:project" ), //
138                                    Type.STRINGS );
139                 NodeBuilder allProps = rules.child( "archiva:projectVersion" ) //
140                     .child( "properties" ) //
141                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
142                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
143                     .setProperty( "indexNodeName", true ) //
144                     .child( "allProps" ) //
145                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
146                 allProps.setProperty( "name", ".*" );
147                 allProps.setProperty( "isRegexp", true );
148                 allProps.setProperty( "nodeScopeIndex", true );
149                 allProps.setProperty( "index", true );
150                 allProps.setProperty( "analyzed", true );
151                 // allProps.setProperty("propertyIndex",true);
152                 allProps = rules.child( "archiva:artifact" ) //
153                     .child( "properties" ) //
154                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
155                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
156                     .setProperty( "indexNodeName", true ).child( "allProps" ) //
157                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
158                 allProps.setProperty( "name", ".*" );
159                 allProps.setProperty( "isRegexp", true );
160                 allProps.setProperty( "nodeScopeIndex", true );
161                 allProps.setProperty( "index", true );
162                 allProps.setProperty( "analyzed", true );
163                 allProps = rules.child( "archiva:facet" ) //
164                     .child( "properties" ) //
165                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
166                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
167                     .setProperty( "indexNodeName", true ) //
168                     .child( "allProps" ) //
169                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
170                 allProps.setProperty( "name", ".*" );
171                 allProps.setProperty( "isRegexp", true );
172                 allProps.setProperty( "nodeScopeIndex", true );
173                 allProps.setProperty( "index", true );
174                 allProps.setProperty( "analyzed", true );
175                 allProps = rules.child( "archiva:namespace" ) //
176                     .child( "properties" ) //
177                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
178                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
179                     .setProperty( "indexNodeName", true ) //
180                     .child( "allProps" ) //
181                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
182                 allProps.setProperty( "name", ".*" );
183                 allProps.setProperty( "isRegexp", true );
184                 allProps.setProperty( "nodeScopeIndex", true );
185                 allProps.setProperty( "index", true );
186                 allProps.setProperty( "analyzed", true );
187                 allProps = rules.child( "archiva:project" ) //
188                     .child( "properties" ) //
189                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) //
190                     .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) //
191                     .setProperty( "indexNodeName", true ) //
192                     .child( "allProps" ) //
193                     .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME );
194                 allProps.setProperty( "name", ".*" );
195                 allProps.setProperty( "isRegexp", true );
196                 allProps.setProperty( "nodeScopeIndex", true );
197                 allProps.setProperty( "index", true );
198                 allProps.setProperty( "analyzed", true );
199 
200                 log.info( "Index: {} myIndex {}", lucene, lucene.getChildNode( "myIndex" ) );
201                 log.info( "myIndex {}", lucene.getChildNode( "myIndex" ).getProperties() );
202                 // IndexUtils.createIndexDefinition(  )
203 
204             }
205         } );
206 
207         StatisticsProvider statsProvider = StatisticsProvider.NOOP;
208         int queueSize = Integer.getInteger( "queueSize", 10000 );
209         Path indexDir = Files.createTempDirectory( "archiva_index" );
210         log.info( "Queue Index {}", indexDir.toString() );
211         IndexCopier indexCopier = new IndexCopier( executorService, indexDir.toFile(), true );
212         NRTIndexFactory nrtIndexFactory = new NRTIndexFactory( indexCopier, statsProvider );
213         MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider();
214         IndexTracker tracker =
215             new IndexTracker( new DefaultIndexReaderFactory( mountInfoProvider, indexCopier ), nrtIndexFactory );
216         DocumentQueue queue = new DocumentQueue( queueSize, tracker, executorService, statsProvider );
217         LocalIndexObserver localIndexObserver = new LocalIndexObserver( queue, statsProvider );
218         LuceneIndexProvider provider = new LuceneIndexProvider( tracker );
219 
220         //        ExternalObserverBuilder builder = new ExternalObserverBuilder(queue, tracker, statsProvider,
221 //            executorService, queueSize);
222 //        Observer observer = builder.build();
223 //        builder.getBackgroundObserver();
224 
225         LuceneIndexEditorProvider editorProvider = //
226             new LuceneIndexEditorProvider( null, tracker, //
227                                            new ExtractedTextCache( 0, 0 ), //
228                                            null, mountInfoProvider );
229         editorProvider.setIndexingQueue( queue );
230 
231         log.info( "Oak: {} with nodeStore {}", oak, nodeStore );
232         Jcr jcr = new Jcr( oak ).with( editorProvider ) //
233             .with( (Observer) provider ) //
234             .with( localIndexObserver )
235             // .with(observer)
236             .with( (QueryIndexProvider) provider ); //
237             //.withAsyncIndexing( "async", 5 );
238         StopWatch stopWatch = new StopWatch();
239         stopWatch.start();
240         Repository r = jcr.createRepository();
241         stopWatch.stop();
242         log.info( "time to create jcr repository: {} ms", stopWatch.getTime() );
243 //        try
244 //        {
245 //            Thread.currentThread().sleep( 1000 );
246 //        }
247 //        catch ( InterruptedException e )
248 //        {
249 //            log.error( e.getMessage(), e );
250 //        }
251         return r;
252 
253 
254     }
255 
256     public void close()
257     {
258         if ( fileStore != null )
259         {
260             fileStore.close();
261         }
262         if (executorService != null)
263         {
264             executorService.shutdownNow();
265         }
266     }
267 
268     public StoreType getStoreType()
269     {
270         return storeType;
271     }
272 
273     public void setStoreType( StoreType storeType )
274     {
275         this.storeType = storeType;
276     }
277 
278     public Path getRepositoryPath()
279     {
280         return repositoryPath;
281     }
282 
283     public void setRepositoryPath( Path repositoryPath )
284     {
285         this.repositoryPath = repositoryPath;
286     }
287 
288     public void setRepositoryPath( String repositoryPath )
289     {
290         this.repositoryPath = Paths.get( repositoryPath );
291         if ( !Files.exists( this.repositoryPath ) )
292         {
293             try
294             {
295                 Files.createDirectories( this.repositoryPath );
296             }
297             catch ( IOException e )
298             {
299                 log.error( e.getMessage(), e );
300                 throw new IllegalArgumentException( "cannot create directory:" + repositoryPath, e );
301             }
302         }
303     }
304 
305     private void createExecutor()
306     {
307         if (executorService ==null )
308         {
309             executorService = Executors.newCachedThreadPool();
310         }
311 
312 //
313 //        ThreadPoolExecutor executor =
314 //            new ThreadPoolExecutor( 0, 5, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(),
315 //                                    new ThreadFactory()
316 //                                    {
317 //                                        private final AtomicInteger counter = new AtomicInteger();
318 //
319 //                                        private final Thread.UncaughtExceptionHandler handler =
320 //                                            new Thread.UncaughtExceptionHandler()
321 //                                            {
322 //                                                @Override
323 //                                                public void uncaughtException( Thread t, Throwable e )
324 //                                                {
325 //                                                    log.warn( "Error occurred in asynchronous processing ", e );
326 //                                                }
327 //                                            };
328 //
329 //                                        @Override
330 //                                        public Thread newThread( @Nonnull Runnable r )
331 //                                        {
332 //                                            Thread thread = new Thread( r, createName() );
333 //                                            thread.setDaemon( true );
334 //                                            thread.setPriority( Thread.MIN_PRIORITY );
335 //                                            thread.setUncaughtExceptionHandler( handler );
336 //                                            return thread;
337 //                                        }
338 //
339 //                                        private String createName()
340 //                                        {
341 //                                            return "oak-lucene-" + counter.getAndIncrement();
342 //                                        }
343 //                                    } );
344 //        executor.setKeepAliveTime( 1, TimeUnit.MINUTES );
345 //        executor.allowCoreThreadTimeOut( true );
346 //        return executor;
347     }
348 
349 }