New parameter support

- chnages variable to define entityID
- added support for IdP metadata from file
- sessions variable control
- attributeresolver/transform/regex plugins support
- sessionsstorage  control

Related-Bug: PROD-20051

Change-Id: I26f8113363f0db4f266277c5cc7405e161fd7a09
diff --git a/README.rst b/README.rst
index daff9cb..5fe15d6 100644
--- a/README.rst
+++ b/README.rst
@@ -12,9 +12,10 @@
   shibboleth:
     server:
       enabled: true
-      keystone_protocol: http
-      keystone_public_address: ${_param:proxy_vip_address_public}
-      keystone_port: 5000
+      app:
+        entity_id: http://${_param:proxy_vip_address_public}:5000
+        signing: false
+        encryption: false
       idp_url: "https://saml.example.com/oam/fed"
       idp_metadata_url: "https://saml.example.com/oamfed/idp/metadata"
       attributes:
@@ -54,4 +55,136 @@
   shibboleth:
     server:
       enabled: true
-      proxy: http://10.10.10.12:8888
\ No newline at end of file
+      proxy: http://10.10.10.12:8888
+
+
+Override IdP metadata from file
+==============
+Sometimes the metadata is not publicly aviailable from IPD. You can define the metadata in pillar. In this case the idp_metadata_url parameter will be ignored.
+
+.. code-block:: yaml
+
+  shibboleth:
+    server:
+      idp_metadata_file: |
+        <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+        <EntityDescriptor xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance" xmlns="urn:oasis:names:tc:SAML:2.0:metadata"
+        entityID="idp_url">
+        <IDPSSODescriptor
+        WantAuthnRequestsSigned="false"
+        protocolSupportEnumeration="urn:oasis:names:tc:SAML:2.0:protocol">
+        <KeyDescriptor use="signing">
+        <ds:KeyInfo xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
+        <ds:X509Data>
+        <ds:X509Certificate>MIIEADi........==</ds:X509Certificate>
+        </ds:X509Data>
+        </ds:KeyInfo>
+        </KeyDescriptor>
+        <KeyDescriptor use="signing">
+        <ds:KeyInfo xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
+        <ds:X509Data>
+        <ds:X509Certificate>MIIEADi........==</ds:X509Certificate>
+        </ds:X509Data>
+        </ds:KeyInfo>
+        </KeyDescriptor>
+        <!-- Supported Name Identifier Formats -->
+        <NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:transient</NameIDFormat>
+        <!-- AuthenticationRequest Consumer endpoint -->
+        <SingleSignOnService
+        Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST"
+        Location="https://IDP_URL/SAMLLogin"
+        />
+        </IDPSSODescriptor>
+        </EntityDescriptor>
+
+
+Shibboleth session control
+==============
+Sometimes there is needed to tune session settings for the application. This has to be done via setting sessions variables Shibboleth2.xml configuration file.
+
+.. code-block:: yaml
+
+  shibboleth:
+    server:
+      sessions:
+        lifetime: 28800
+        timeout: 3600
+        relaystate: "ss:mem"
+        checkaddress: "false"
+        handlerssl: "false"
+        cookieprops: "http"
+
+
+Shibboleth attributeresolver/regex plugins support
+==============
+Sometimes there is needed to set add new attribute by extracting some information from other attributes.  This has to be done loading the plugin and a adding attributeresolver with transform type in Shibboleth2.xml configuration file.
+See more detail here: https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPAttributeResolver#NativeSPAttributeResolver-TransformAttributeResolver(Version2.5andAbove)
+
+.. code-block:: yaml
+
+  shibboleth:
+    server:
+      outofprocess:
+        extensions:
+          library:
+            plugin1:
+               path: plugins.so
+               fatal: "true"
+      attributeresolver:
+        transform:
+          Email:
+            mantch1:
+              match: "@.*$"
+              destination_name: "User-identifier"
+              destination: "$1"
+            mantch2:
+              match: "@.*$"
+              destination: "$2"
+Shibboleth shared session
+==============
+Sometimes there is needed to set shibd on each controller where keystone is running. To make sure sessions are accessible and shared between all of them you need to setup shared storage for sessions
+The example below shows you how to setup shared storage using memcached available on controllers:
+Please note that sessioncache requires memcached with bitmap set to true. Omitting sessioncache element will result in an in-memory plugin identified as id="mem".
+https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPStorageService
+
+.. code-block:: yaml
+
+  shibboleth:
+    server:
+      outofprocess:
+        extensions:
+          library:
+            plugin1:
+              path:  "memcache-store.so"
+              fatal: "true"
+      storageservice:
+        mc:
+          type: MEMCACHE
+          buildmap: "0"
+          sendtimeout: "999999" #optional
+          recvtimeout: "999999" #optional
+          polltimeout: "1000" #optional
+          failtimeout: "5" #optional
+          retrytimeout: "30" #optional
+          prefix: "SHIBD" #optional
+          hosts: "${_param:cluster_node01_address}:11211,${_param:cluster_node02_address}:11211,${_param:cluster_node03_address}:11211
+        mc-ctx:
+          type: MEMCACHE
+          buildmap: "1"
+          sendtimeout: "999999" #optional
+          recvtimeout: "999999" #optional
+          polltimeout: "1000" #optional
+          failtimeout: "5" #optional
+          retrytimeout: "30" #optional
+          prefix: "SHIBD" #optional
+          hosts: "${_param:cluster_node01_address}:11211,${_param:cluster_node02_address}:11211,${_param:cluster_node03_address}:11211
+      sessioncache:
+        type: "StorageService"
+        cachetimeout: "900" #optional
+        storageservice: "mc-ctx"
+        storageservicelite: "mc"
+      replaycache:
+        storageservice: "mc"
+      replaycache:
+        storageservice: "mc"
+        artifactTTL: "180"  #optional
diff --git a/metadata/service/server/cluster.yml b/metadata/service/server/cluster.yml
index 24cbef0..53de089 100644
--- a/metadata/service/server/cluster.yml
+++ b/metadata/service/server/cluster.yml
@@ -5,9 +5,9 @@
 parameters:
   shibboleth:
     server:
-      enabled: true
-      keystone_protocol: http
-      keystone_public_address: ${_param:proxy_vip_address_public}
-      keystone_port: 5000
+      app:
+        entity_id: http://${_param:proxy_vip_address_public}:5000
+        signing: "false"
+        encryption: "false"
       idp_url: "https://saml.example.com/oam/fed"
       idp_metadata_url: "https://saml.example.com/oamfed/idp/metadata"
diff --git a/metadata/service/server/single.yml b/metadata/service/server/single.yml
index 24cbef0..8905ce7 100644
--- a/metadata/service/server/single.yml
+++ b/metadata/service/server/single.yml
@@ -6,8 +6,9 @@
   shibboleth:
     server:
       enabled: true
-      keystone_protocol: http
-      keystone_public_address: ${_param:proxy_vip_address_public}
-      keystone_port: 5000
+      app:
+        entity_id: http://${_param:proxy_vip_address_public}:5000
+        signing: "false"
+        encryption: "false"
       idp_url: "https://saml.example.com/oam/fed"
       idp_metadata_url: "https://saml.example.com/oamfed/idp/metadata"
diff --git a/shibboleth/files/shibboleth2.xml b/shibboleth/files/shibboleth2.xml
index f900165..eec2dad 100644
--- a/shibboleth/files/shibboleth2.xml
+++ b/shibboleth/files/shibboleth2.xml
@@ -19,10 +19,51 @@
     For examples with the RequestMap XML syntax instead, see the example-shibboleth2.xml
     file, and the https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPRequestMapHowTo topic.
     -->
-
+    {%- if server.outofprocess is defined %}
+    <OutOfProcess>
+    {%- if server.outofprocess.extensions is defined %}
+      <Extensions>
+    {%- if server.outofprocess.extensions.library is defined %}
+       {%- for name, plugin in server.get('outofprocess', {}).get('extensions', {}).get('library', {}).iteritems() %}
+        <Library path="{{ plugin.get('path', '') }}" fatal="{{ plugin.get('fatal', "true") }}"/>
+       {%- endfor %}
+    {%- endif %}
+      </Extensions>
+    {%- endif %}
+    </OutOfProcess>
+    {%- endif %}
+    {%- if server.storageservice is defined %}
+      {%- for id, attributes in server.get('storageservice').iteritems() %}
+        {%- if attributes.type == "MEMCACHE" %}
+     <StorageService type="MEMCACHE" id="{{ id }}" buildMap="{{ attributes.get('buildmap', "0") }}" sendTimeout="{{ attributes.get('sendtimeout', "999999") }}"
+                recvTimeout="{{ attributes.get('recvtimeout', "999999") }}" pollTimeout="{{ attributes.get('polltimeout', "1000") }}"
+                failLimit="{{ attributes.get('failtimeout', "5") }}"
+                prefix="{{ attributes.get('prefix', "SHIBD") }}" retryTimeout="{{ attributes.get('retrytimeout', "30") }}">
+       <Hosts>
+        {{ attributes.hosts }}
+       </Hosts>
+     </StorageService>
+        {%- elif server.attributes.type == "Memory" %}
+     <StorageService type="Memory" id="{{ id }}" cleanupInterval="{{ attributes.get('cleanupinterval', "900") }}"/>
+        {%- endif %}
+      {% endfor %}
+    {%- endif %}
+    {%- if server.sessioncache is defined %}
+    <SessionCache type="{{ server.sessioncache.get('type', "StorageService") }}" cacheTimeout="{{ server.sessioncache.get('cachetimeout', "900") }}"
+      StorageService="{{ server.sessioncache.get('storageservice', "mem") }}" {%- if server.sessioncache.storageservicelite is defined %}StorageServiceLite="{{ server.sessioncache.get('storageservicelite', "mem") }}"{%- endif %} />
+    {%- endif %}
+    {%- if server.replaycache is defined %}
+    <ReplayCache StorageService="{{ server.replaycache.get('storageservice', "mem") }}"/>
+    {%- endif %}
+    {%- if server.artifactmap is defined %}
+    <ArtifactMap StorageService="{{ server.artifactmap.get('storageservice', "mem") }}" artifactTTL="{{ server.artifactmap.get('artifactttl', "180") }}" />
+    {%- endif %}
     <!-- The ApplicationDefaults element is where most of Shibboleth's SAML bits are defined. -->
+    {%- if server.app.entity_id is defined %}
+    <ApplicationDefaults entityID="{{ server.app.entity_id }}" signing="{{ server.app.signing }}" encryption="{{ server.app.encryption }}">
+    {%- else %}
     <ApplicationDefaults entityID="{{ server.keystone_protocol }}://{{ server.keystone_public_address }}:{{ server.keystone_port }}">
-
+    {%- endif %}
         <!--
         Controls session lifetimes, address checks, cookie handling, and the protocol handlers.
         You MUST supply an effectively unique handlerURL value for each of your applications.
@@ -32,8 +73,10 @@
         Note that while we default checkAddress to "false", this has a negative impact on the
         security of your site. Stealing sessions via cookie theft is much easier with this disabled.
         -->
-        <Sessions lifetime="28800" timeout="3600" relayState="ss:mem"
-                  checkAddress="false" handlerSSL="false" cookieProps="http">
+        <Sessions lifetime="{{ server.get('sessions', {}).get('lifetime', 28800) }}" timeout="{{ server.get('sessions', {}).get('timeout', 3600) }}"
+                  relayState="{{ server.get('sessions', {}).get('relaystate', "ss:mem") }}"
+                  checkAddress="{{ server.get('sessions', {}).get('checkaddress', "false") }}" handlerSSL="{{ server.get('sessions', {}).get('handlerssl', "false") }}"
+                  cookieProps="{{ server.get('sessions', {}).get('cookieprops', "http") }}">
 
             <!--
             Configures SSO for a default IdP. To allow for >1 IdP, remove
@@ -69,6 +112,9 @@
             helpLocation="/about.html"
             styleSheet="/shibboleth-sp/main.css"/>
 
+        {%- if server.idp_metadata_file is defined %}
+        <MetadataProvider type="XML" file="/etc/shibboleth/idp-metadata.xml"/>
+        {% elif server.idp_metadata_url is defined %}
         <MetadataProvider type="XML" uri="{{ server.idp_metadata_url }}"
               backingFilePath="/var/cache/shibboleth/metadata.xml" reloadInterval="180000">
             {%- if server.idp_certificate is defined %}
@@ -78,6 +124,7 @@
             <TransportOption provider="CURL" option="10004">"{{ server.proxy }}"</TransportOption>
             {%- endif %}
         </MetadataProvider>
+        {%- endif %}
 
         <!-- Example of locally maintained metadata. -->
         <!--
@@ -90,6 +137,22 @@
         <!-- Use a SAML query if no attributes are supplied during SSO. -->
         <AttributeResolver type="Query" subjectMatch="true"/>
 
+        {%- if server.attributeresolver is defined %}
+        {%- if server.attributeresolver.transform is defined %}
+        {%- for source, regex in server.get('attributeresolver', {}).get('transform', {}).iteritems() %}
+         <AttributeResolver type="Transform" source="{{ source }}">
+        {%- for match, attr in regex.iteritems() %}
+        {%- if attr.destination_name is defined %}
+             <Regex match="{{ attr.get('match', "") }}" dest="{{ attr.get('destination_name', "") }}">{{ attr.get('destination', "") }}</Regex>
+               {% else %}
+             <Regex match="{{ attr.get('match', "") }}" >{{ attr.get('destination', "") }}</Regex>
+            {%- endif %}
+          {%- endfor %}
+          {%- endfor %}
+        </AttributeResolver>
+        {%- endif %}
+        {%- endif %}
+
         <!-- Default filtering policy for recognized attributes, lets other data pass. -->
         <AttributeFilter type="XML" validate="true" path="attribute-policy.xml"/>
 
diff --git a/shibboleth/server.sls b/shibboleth/server.sls
index 4b345e4..87ad1bc 100644
--- a/shibboleth/server.sls
+++ b/shibboleth/server.sls
@@ -15,6 +15,17 @@
     - service: apache_service
     - service: shibboleth_service
 
+{%- if server.idp_metadata_file is defined %}
+/etc/shibboleth/idp-metadata.xml:
+  file.managed:
+  - contents: {{ server.idp_metadata_file | yaml_encode}}
+  - require:
+    - pkg: apache_packages
+  - watch_in:
+    - service: apache_service
+    - service: shibboleth_service
+{%- endif %}
+
 {%- if server.idp_certificate is defined %}
 /etc/shibboleth/fedsigner.pem:
   file.managed:
diff --git a/tests/pillar/shibboleth.sls b/tests/pillar/shibboleth.sls
index ac37774..72b0f2e 100644
--- a/tests/pillar/shibboleth.sls
+++ b/tests/pillar/shibboleth.sls
@@ -1,15 +1,77 @@
 shibboleth:
   server:
     enabled: true
-    keystone_protocol: http
-    keystone_public_address: ${_param:proxy_vip_address_public}
-    keystone_port: 5000
+    app:
+      entity_id: http://${_param:proxy_vip_address_public}:5000
+      signing: false
+      encryption: false
     idp_url: "https://saml.example.com/oam/fed"
     idp_metadata_url: "https://saml.example.com/oamfed/idp/metadata"
     attributes:
     - name: test
       id: test
       name_format: urn:oasis:names:tc:SAML:2.0:attrname-format:basic
+    idp_metadata_file: |
+      <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+      <EntityDescriptor xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance" xmlns="urn:oasis:names:tc:SAML:2.0:metadata"
+      entityID="idp_url">
+      <IDPSSODescriptor
+      </IDPSSODescriptor>
+      </EntityDescriptor>
+    sessions:
+      lifetime: 28800
+      timeout: 3600
+      relaystate: "ss:mem"
+      checkaddress: "false"
+      handlerssl: "false"
+      cookieprops: "http"
+    outofprocess:
+      extensions:
+        library:
+          plugin1:
+            path:  "memcache-store.so"
+            fatal: "true"
+    storageservice:
+      mc:
+        type: MEMCACHE
+        buildmap: "0"
+        sendtimeout: "999999" #optional
+        recvtimeout: "999999" #optional
+        polltimeout: "1000" #optional
+        failtimeout: "5" #optional
+        retrytimeout: "30" #optional
+        prefix: "SHIBD" #optional
+        hosts: "127.0.0.1:11211"
+      mc-ctx:
+        type: MEMCACHE
+        buildmap: "1"
+        sendtimeout: "999999" #optional
+        recvtimeout: "999999" #optional
+        polltimeout: "1000" #optional
+        failtimeout: "5" #optional
+        retrytimeout: "30" #optional
+        prefix: "SHIBD" #optional
+        hosts: "127.0.0.1:11211"
+    sessioncache:
+      type: "StorageService"
+      cachetimeout: "900" #optional
+      storageservice: "mc-ctx"
+      storageservicelite: "mc"
+    replaycache:
+      storageservice: "mc"
+    replaycache:
+      storageservice: "mc"
+      artifactTTL: "180"  #optional
+    attributeresolver:
+      transform:
+        Email:
+          mantch1:
+            match: "@.*$"
+            destination_name: "User-identifier"
+            destination: "$1"
+          mantch2:
+            match: "@.*$"
+            destination: "$2"
 apache:
   server:
     enabled: true