From: Wayne Schneider Date: Fri, 12 Aug 2016 23:32:30 +0000 (-0500) Subject: More work towards SolrCloud/Docker. X-Git-Url: http://git.indexdata.com/?p=lui-solr.git;a=commitdiff_plain;h=664635ceab85dcb4b9624bcce816a3da51e00b26 More work towards SolrCloud/Docker. --- diff --git a/docker/docker-deploy.yml b/docker/docker-deploy.yml index 628095a..ca0d4ab 100644 --- a/docker/docker-deploy.yml +++ b/docker/docker-deploy.yml @@ -43,28 +43,21 @@ become: yes service: name=docker enabled=yes state=started - - name: Check for luinet network - command: docker network ls -f name=luinet -q - register: luinet - - - name: Create luinet network - command: docker network create luinet - when: luinet.stdout == "" - - name: Launch ZooKeeper + become: yes docker_container: name=zk1 image=jplock/zookeeper - - name: Get networks for ZooKeeper - shell: "docker inspect --format={% raw %}'{{json .NetworkSettings.Networks}}'{% endraw %} zk1" - register: zk1_networks + - name: Get contents of /solr path in ZooKeeper + become: yes + command: docker exec -t zk1 bin/zkCli.sh get /solr + register: zk1_solr - - name: Attach ZooKeeper to luinet network - command: docker network connect luinet zk1 - when: (zk1_networks.stdout|from_json).luinet is not defined + - name: Create /solr path in ZooKeeper + become: yes + command: docker exec -t zk1 bin/zkCli.sh create /solr [] + when: zk1_solr.stdout.find('Node does not exist') != -1 - - name: Detach ZooKeeper from bridge network - command: docker network disconnect bridge zk1 - when: (zk1_networks.stdout|from_json).bridge is defined +# Separate paths for setting up Solr nodes - hosts: dev roles: @@ -74,25 +67,41 @@ roles: - prod -- hosts: all +# Back to everyone +- hosts: all tasks: - - name: Get networks for Solr - shell: "docker inspect --format={% raw %}'{{json .NetworkSettings.Networks}}'{% endraw %} solr1" - register: solr1_networks - - - name: Attach Solr to luinet network - command: docker network connect luinet solr1 - when: (solr1_networks.stdout|from_json).luinet is not defined - - - name: Detach Solr from bridge network - command: docker network disconnect bridge solr1 - when: (solr1_networks.stdout|from_json).bridge is defined - - # - name: Create Solr container - # docker_container: - # name: solr1 - # tty: yes - # published_ports: 8983:8983 - # #volumes: /vagrant/conf/solr/solr-home:/opt/solr/server/solr - # command: bash -c '/opt/solr/bin/solr start -f -z zk1:2181' + - name: Launch Solr1 + become: yes + docker_container: + name: solr1 + image: lui-solr + tty: yes + published_ports: 8983:8983 + links: zk1:ZK1 + command: bash -c '/opt/solr/bin/solr start -f -z $ZK1_PORT_2181_TCP_ADDR:$ZK1_PORT_2181_TCP_PORT/solr' + + - name: Launch Solr2 + become: yes + docker_container: + name: solr2 + image: lui-solr + tty: yes + published_ports: 8984:8983 + links: zk1:ZK1 + command: bash -c '/opt/solr/bin/solr start -f -z $ZK1_PORT_2181_TCP_ADDR:$ZK1_PORT_2181_TCP_PORT/solr' + + ### HERE I AM ### + # wait for ZooKeeper to see all the solr nodes before proceeding + # look at Ansible until module + + - name: Check lui collection + become: yes + command: docker exec -t zk1 bin/zkCli.sh ls /solr/collections + register: lui_collection + + - name: Create lui collection + become: yes + command: docker exec -t solr1 bin/solr create -c lui -d /opt/solr/lui-solr -shards 2 + when: (lui_collection.stdout_lines|last).find('lui') == -1 + diff --git a/docker/roles/dev/tasks/main.yml b/docker/roles/dev/tasks/main.yml index 80fff39..5fd2412 100644 --- a/docker/roles/dev/tasks/main.yml +++ b/docker/roles/dev/tasks/main.yml @@ -1,13 +1,4 @@ --- - name: Build the lui-solr image + become: yes docker_image: name=lui-solr path=/vagrant/docker/solr - -- name: Create Solr container - docker_container: - name: solr1 - image: lui-solr - tty: yes - published_ports: 8983:8983 - #volumes: /vagrant/conf/solr/solr-home:/opt/solr/server/solr - command: bash -c '/opt/solr/bin/solr start -f -z zk1:2181' - state: present diff --git a/docker/solr/Dockerfile b/docker/solr/Dockerfile index e289f80..f40a78b 100644 --- a/docker/solr/Dockerfile +++ b/docker/solr/Dockerfile @@ -1,6 +1,4 @@ -# Create Solr data directory as expected by lui-solr solrconfig.xml +# Copy lui-solr config FROM solr:6.1.0 MAINTAINER Index Data -USER root -RUN mkdir -p /var/lib/masterkey/lui/solr/lui && chown -R solr:solr /var/lib/masterkey/lui/solr -USER solr \ No newline at end of file +COPY lui-solr lui-solr \ No newline at end of file diff --git a/docker/solr/lui-solr/admin-extra.html b/docker/solr/lui-solr/admin-extra.html new file mode 100644 index 0000000..fecab20 --- /dev/null +++ b/docker/solr/lui-solr/admin-extra.html @@ -0,0 +1,24 @@ + + + diff --git a/docker/solr/lui-solr/admin-extra.menu-bottom.html b/docker/solr/lui-solr/admin-extra.menu-bottom.html new file mode 100644 index 0000000..3359a46 --- /dev/null +++ b/docker/solr/lui-solr/admin-extra.menu-bottom.html @@ -0,0 +1,25 @@ + + + + diff --git a/docker/solr/lui-solr/admin-extra.menu-top.html b/docker/solr/lui-solr/admin-extra.menu-top.html new file mode 100644 index 0000000..0886cee --- /dev/null +++ b/docker/solr/lui-solr/admin-extra.menu-top.html @@ -0,0 +1,25 @@ + + + + diff --git a/docker/solr/lui-solr/protwords.txt b/docker/solr/lui-solr/protwords.txt new file mode 100644 index 0000000..1dfc0ab --- /dev/null +++ b/docker/solr/lui-solr/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/docker/solr/lui-solr/schema.xml b/docker/solr/lui-solr/schema.xml new file mode 100644 index 0000000..9253610 --- /dev/null +++ b/docker/solr/lui-solr/schema.xml @@ -0,0 +1,271 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + diff --git a/docker/solr/lui-solr/solrconfig.xml b/docker/solr/lui-solr/solrconfig.xml new file mode 100644 index 0000000..89f91b9 --- /dev/null +++ b/docker/solr/lui-solr/solrconfig.xml @@ -0,0 +1,267 @@ + + + + + 5.5.1 + + + + + + + + ${solr.lock.type:native} + + true + + + + + + + + + + + ${solr.ulog.dir:} + + + + ${solr.autoCommit.maxTime:15000} + false + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + 1024 + + + + + + + + true + 20 + 200 + + + + + + database:* + true + 1 + author_exact + subject_exact + medium_exact + date + database + + + + + + + + database:* + true + 1 + author_exact + subject_exact + medium_exact + date + database + + + + + false + 2 + + + + + + + + + + + + + + + + explicit + 10 + text + *,score + AND + + + + + + explicit + json + true + text + *,score + AND + + + + + + true + json + true + + + + + + {!xport} + xsort + false + + + + query + + + + + + + + clone-fields + + + + + + + + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + + + + solrpingquery + + + all + + + + + + + explicit + true + + + + + + + commit + startup + schema.xml,stopwords.txt + + + + + + + + + + .* + + .*_binary$ + + + text + + + + + + + + + + text/plain; charset=UTF-8 + + + + + *:* + + + diff --git a/docker/solr/lui-solr/stopwords.txt b/docker/solr/lui-solr/stopwords.txt new file mode 100644 index 0000000..b5824da --- /dev/null +++ b/docker/solr/lui-solr/stopwords.txt @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + diff --git a/docker/solr/lui-solr/synonyms.txt b/docker/solr/lui-solr/synonyms.txt new file mode 100644 index 0000000..b0e31cb --- /dev/null +++ b/docker/solr/lui-solr/synonyms.txt @@ -0,0 +1,31 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaa => aaaa +bbb => bbbb1 bbbb2 +ccc => cccc1,cccc2 +a\=>a => b\=>b +a\,a => b\,b +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma +