Automate check-in of development catalog items to SCM

Owned by Chandresh Patel
Last updated: Aug 06, 2019

Oracle BI projects can either source catalog objects from SCM or Development servers, i.e you will either have Populate from Git or Populate from OBIEE Web Catalog on files tab of your project. Similar to other technologies, FlexDeploy can sync changes on development server to SCM. This helps implement DevOps best practice to source control developer code. Due to inherent differences in naming of objects (mainly file extensions), this requires some additional steps in workflow.

Your project for this purpose may look something like this.

Similar to other projects, you will setup project properties to indicate folder that you will be working with from catalog. Our purpose is to extract files from development server and commit to Git with necessary changes in file and/or extension names. Note that you are using this as part of build workflow. You can implement one of two options - 1) standard build/deploy from development server with check-in to SCM for historical reference. or 2) standard build/deploy from SCM with this project executing extract from Development and check-in to SCM. With either option, you end up with code being managed in SCM of your choice.

Now let's talk about Extract WebCatalog workflow. This workflow will execute 6 steps as shown below.

  1. Extract from development environment. Based on whether All Files or Package build is executing, number of objects will be extracted.
  2. Clone git repository where objects are stored.
  3. Copy files extracted in step 1 to cloned repository.
  4. Add any new files in local git repository.
  5. Commit any changes to local git repository. It is possible that nothing is found to commit if there were no changes on development server.
  6. Push changes to remote repository.

Here is source code for the workflow. You will need to update this workflow to indicate project Git instance code and sparse checkout folder for catalog objects. Also, keep in mind that we are running this build against master branch(stream), if you want to commit on different branch then create stream on project appropriately.

<?xml version="1.0" encoding="UTF-8"?>
<ns0:WorkFlow xmlns:ns1="http://flexagon.com/flexdeploy/workflow/common" xmlns:ns0="http://flexagon.com/flexdeploy/workflow">
   <ns1:Name>Extract WebCatalog</ns1:Name>
   <ns1:Description></ns1:Description>
   <ns0:Steps>
      <ns0:Step>
         <ns1:Name>obiBuild</ns1:Name>
         <ns1:StepId>1</ns1:StepId>
         <ns0:InvokePlugin>
            <ns0:PluginName>FlexagonOBIEEPlugin</ns0:PluginName>
            <ns0:PluginOperation>obiBuild</ns0:PluginOperation>
            <ns0:consumesArtifacts>false</ns0:consumesArtifacts>
            <ns0:producesArtifacts>false</ns0:producesArtifacts>
            <ns0:PluginInputs>
               <ns0:PluginInput>
                  <ns0:Name>FDOBIEE_INP_COPY_PERMISSIONS</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>Boolean</ns1:Type>
                  <ns0:Text>true</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDOBIEE_INP_RECURSIVE_CAT_BUILD</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>Boolean</ns1:Type>
                  <ns0:Text>true</ns0:Text>
               </ns0:PluginInput>
            </ns0:PluginInputs>
            <ns0:UserInputs/>
            <ns0:PluginOutputs/>
            <ns0:UserOutputs/>
         </ns0:InvokePlugin>
      </ns0:Step>
      <ns0:Step>
         <ns1:Name>Git Clone</ns1:Name>
         <ns1:StepId>2</ns1:StepId>
         <ns0:InvokePlugin>
            <ns0:PluginName>FlexagonGITPlugin</ns0:PluginName>
            <ns0:PluginOperation>clone</ns0:PluginOperation>
            <ns0:consumesArtifacts>false</ns0:consumesArtifacts>
            <ns0:producesArtifacts>false</ns0:producesArtifacts>
            <ns0:PluginInputs>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_INSTANCE_CODE</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>GITDEMO</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_DEST_SUBFOLDER</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>../obieefiles</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_BRANCH</ns0:Name>
                  <ns0:ValueType>Expression</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Value>
                     <ns0:Expression>FDBLD_STREAM_NAME</ns0:Expression>
                  </ns0:Value>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_DEPTH</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>Integer</ns1:Type>
                  <ns0:Text>1</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_SPARSE_CHECKOUT_FOLDERS</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>OBIEECatalog</ns0:Text>
               </ns0:PluginInput>
            </ns0:PluginInputs>
            <ns0:UserInputs/>
            <ns0:PluginOutputs/>
            <ns0:UserOutputs/>
         </ns0:InvokePlugin>
      </ns0:Step>
      <ns0:Step>
         <ns1:Name>Copy downloaded files to Git Repository</ns1:Name>
         <ns1:StepId>3</ns1:StepId>
         <ns0:InvokePlugin>
            <ns0:PluginName>FlexagonShellPlugin</ns0:PluginName>
            <ns0:PluginOperation>partialExecute</ns0:PluginOperation>
            <ns0:consumesArtifacts>false</ns0:consumesArtifacts>
            <ns0:producesArtifacts>false</ns0:producesArtifacts>
            <ns0:EndpointSelection>
               <ns0:EndpointSelectionChoice>All</ns0:EndpointSelectionChoice>
            </ns0:EndpointSelection>
            <ns0:EndpointExecutionChoice>Any</ns0:EndpointExecutionChoice>
            <ns0:PluginInputs>
               <ns0:PluginInput>
                  <ns0:Name>FDSHELL_INP_CODE_SNIPPET</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text># copy files from FD_TEMP_DIR to FD_TEMP_DIR + '/../obieefiles/OBIEECatalog'
# copy and at same time apply extension to file also.

SOURCE_FILE_MODIFIED=`echo "${SOURCE_FILE/:/%3a}"`
EXT=$FILE_EXTENSION
SOURCE_EXT_WITH_DOT=""
if [ "$OBJECT_TYPE_CODE" = "FOLDER" ]
then
    EXT="fldr"
    SOURCE_EXT_WITH_DOT=".catalog"
fi
if [ "$OBJECT_TYPE_CODE" = "ANALYSIS" ]
then
    EXT="anlys"
fi
if [ "$OBJECT_TYPE_CODE" = "DASHBOARD" ]
then
    EXT="db"
fi
if [ "$OBJECT_TYPE_CODE" = "DASHBOARD_PROMPT" ]
then
    EXT="dbp"
fi
if [ "$OBJECT_TYPE_CODE" = "REPORT" ]
then
    EXT="rprt"
fi
if [ "$OBJECT_TYPE_CODE" = "DATAMODEL" ]
then
    EXT="dm"
fi
if [ "$OBJECT_TYPE_CODE" = "STYLE_TEMPLATE" ]
then
    EXT="st"
fi
if [ "$OBJECT_TYPE_CODE" = "METADATA" ]
then
    EXT="md"
fi
if [ "$OBJECT_TYPE_CODE" = "KPI" ]
then
    EXT="kpi"
fi
if [ "$OBJECT_TYPE_CODE" = "FILTER" ]
then
    EXT="fltr"
fi
mkdir -p "$FD_TEMP_DIR/../obieefiles/OBIEECatalog$FILE_PATH_PREFIX"
TARGET_FILE=$FD_TEMP_DIR/../obieefiles/OBIEECatalog$FILE_PATH_PREFIX/$FILE_NAME.$EXT
if [ "$OBJECT_TYPE_CODE" = "OTHER" ]
then
    TARGET_FILE=$FD_TEMP_DIR/../obieefiles/OBIEECatalog$FILE_PATH_PREFIX/$FILE_NAME
fi
echo "Source file is $SOURCE_FILE_MODIFIED$SOURCE_EXT_WITH_DOT"
echo "Target file is $TARGET_FILE"
cp "$SOURCE_FILE_MODIFIED$SOURCE_EXT_WITH_DOT" "$TARGET_FILE"</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDSHELL_INP_STOP_ON_ERROR</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>Boolean</ns1:Type>
                  <ns0:Text>true</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDSHELL_INP_DISABLE_ECHO</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>Boolean</ns1:Type>
                  <ns0:Text>true</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDSHELL_INP_RESTRICT_ENVIRONMENT</ns0:Name>
                  <ns0:ValueType>Expression</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Value>
                     <ns0:Expression>FD_INSTANCE_CODE</ns0:Expression>
                  </ns0:Value>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDSHELL_INP_NO_SECURE_VARIABLES</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>Boolean</ns1:Type>
                  <ns0:Text>true</ns0:Text>
               </ns0:PluginInput>
            </ns0:PluginInputs>
            <ns0:UserInputs/>
            <ns0:PluginOutputs/>
            <ns0:UserOutputs/>
         </ns0:InvokePlugin>
      </ns0:Step>
      <ns0:Step>
         <ns1:Name>Add any new Files</ns1:Name>
         <ns1:StepId>4</ns1:StepId>
         <ns0:InvokePlugin>
            <ns0:PluginName>FlexagonGITPlugin</ns0:PluginName>
            <ns0:PluginOperation>add</ns0:PluginOperation>
            <ns0:consumesArtifacts>false</ns0:consumesArtifacts>
            <ns0:producesArtifacts>false</ns0:producesArtifacts>
            <ns0:PluginInputs>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_INSTANCE_CODE</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>GITDEMO</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_DEST_SUBFOLDER</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>../obieefiles</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_FILELIST</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>--all .</ns0:Text>
               </ns0:PluginInput>
            </ns0:PluginInputs>
            <ns0:UserInputs/>
            <ns0:PluginOutputs/>
            <ns0:UserOutputs/>
         </ns0:InvokePlugin>
      </ns0:Step>
      <ns0:Step>
         <ns1:Name>Commit to Git</ns1:Name>
         <ns1:StepId>5</ns1:StepId>
         <ns0:InvokePlugin>
            <ns0:PluginName>FlexagonGITPlugin</ns0:PluginName>
            <ns0:PluginOperation>commit</ns0:PluginOperation>
            <ns0:consumesArtifacts>false</ns0:consumesArtifacts>
            <ns0:producesArtifacts>false</ns0:producesArtifacts>
            <ns0:PluginInputs>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_INSTANCE_CODE</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>GITDEMO</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_DEST_SUBFOLDER</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>../obieefiles</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_NAME</ns0:Name>
                  <ns0:ValueType>Expression</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Value>
                     <ns0:Expression>FD_WF_USER</ns0:Expression>
                  </ns0:Value>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_EMAIL</ns0:Name>
                  <ns0:ValueType>Expression</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Value>
                     <ns0:Expression>FD_WF_USER_EMAIL</ns0:Expression>
                  </ns0:Value>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_REVISION_MESSAGE</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>new obiee files</ns0:Text>
               </ns0:PluginInput>
            </ns0:PluginInputs>
            <ns0:UserInputs/>
            <ns0:PluginOutputs/>
            <ns0:UserOutputs/>
         </ns0:InvokePlugin>
      </ns0:Step>
      <ns0:Step>
         <ns1:Name>Push to Central Repository</ns1:Name>
         <ns1:StepId>6</ns1:StepId>
         <ns0:InvokePlugin>
            <ns0:PluginName>FlexagonGITPlugin</ns0:PluginName>
            <ns0:PluginOperation>push</ns0:PluginOperation>
            <ns0:consumesArtifacts>false</ns0:consumesArtifacts>
            <ns0:producesArtifacts>false</ns0:producesArtifacts>
            <ns0:PluginInputs>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_INSTANCE_CODE</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>GITDEMO</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_DEST_SUBFOLDER</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Text>../obieefiles</ns0:Text>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_BRANCH</ns0:Name>
                  <ns0:ValueType>Expression</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>String</ns1:Type>
                  <ns0:Value>
                     <ns0:Expression>FDBLD_STREAM_NAME</ns0:Expression>
                  </ns0:Value>
               </ns0:PluginInput>
               <ns0:PluginInput>
                  <ns0:Name>FDGIT_INP_FOLLOW_TAGS</ns0:Name>
                  <ns0:ValueType>Text</ns0:ValueType>
                  <ns0:Encrypted>false</ns0:Encrypted>
                  <ns1:Type>Boolean</ns1:Type>
                  <ns0:Text>false</ns0:Text>
               </ns0:PluginInput>
            </ns0:PluginInputs>
            <ns0:UserInputs/>
            <ns0:PluginOutputs/>
            <ns0:UserOutputs/>
         </ns0:InvokePlugin>
      </ns0:Step>
   </ns0:Steps>
</ns0:WorkFlow>
The following macros are not currently supported in the footer:
  • style