NAME
OcToolkit - Open Cloud Toolkit
SYNOPSIS
use OcToolkit;
my $ocObj = OcToolkit->new(
advanceFeatures => $advanceFeatures,
clusterBaseAddress => $clusterBaseAddress,
cluster => $cluster,
ocConfigFile => $ocConfigFile,
host => $host,
ocResourceKinds => $ocResourceKinds,
componentDirs => $componentDirs,
namespace => $namespace,
projectName => $projectName,
omit => $omit,
urlPrefix => $urlPrefix,
clusterIpRange => $clusterIpRange,
secretsDir => $secretsDir,
sortType => $sortType,
templatesTTDir => $templatesTTDir,
yamlToTTconvertDir => $yamlToTTconvertDir,
specificYamlFile => $specificYamlFile,
templatesYamlDir => $templatesYamlDir,
addFlagValuesToConfig => \&addFlagValuesToConfig,
componentIsAllowed => \&componentIsAllowed,
generateUrl => \&generateUrl,
removeClutter => \&removeClutter,
removeClutterBackup => \&removeClutterBackup);
$ocObj->install('test');
$ocObj->validate('test');
$ocObj->update('test');
$ocObj->backup('prod');
$ocObj->delete('dev');DESCRIPTION
Helm like tool for Openshift and Kubernetes with multi cluster support. See https://gitlab.com/code7143615/octoolkit/-/blob/master/README.md how to use this library in ocToolkit.pl script and use it as 'Helm-like' command line tool
LICENSE
Copyright (C) John Summers.
This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself.
AUTHOR
John Summers <devp2000a@gmail.com>
SCRIPT TO RUN IT
#!/usr/bin/perl
use strict; use warnings;
# search for Module in same directory where this script is use FindBin; use File::Spec; use lib File::Spec->catdir($FindBin::Bin);
use Getopt::Std; use OcToolkit; use Data::Dumper;
my $help = "Install/uninstall/backup/validate/upgrade instances and they components into/from Openshift/Kubernetes cluster Put your templates in 'templates_tt' dir located in current dir. Every component should have own dir(10-api,20-solr,...) inside of it. Write your templates with Template Toolkit(https://template-toolkit.org) templating engine. Put your config into oc_config.json. Located in current dir. 'instance_specific_data' : put your instance specific data in this json node, see example belove and in git 'instance_specific_name' : instance string will be automatically added at the end of every entry, see example belove and in git 'project': project specifc data oc_config.json root data are fix for every instance
Flags description: -A cluster base address, e.g.: 'apps.clusterintern' or 'apps.clusterpublic', used in default route url generation -a advance features: '-a multipleClusters' if cluster specific templates are required, put them into nested dir e.g.: templates_tt/50-api/clusterIntern/50-deployment-config-api.tt, ocToolkit will automatically select corresponding templates (currently this flag is selected by default) '-a kubectl' use Kubernetes 'kubectl' instead of default Openshift 'oc' command '-a removeClutter' remove clutter yaml fields during backup Use multiple: -a 'multipleClusters;kubectl;removeClutter' -b instance name(s), backups specific instance(s) sorted by components e.g.: -b 'dev;test' or -b all to backup whole project(unsorted) if '-a multipleClusters' and not '-b all' is used then '-c' flag is mandatory -c cluster name e.g.: 'clusterIntern','clusterPublic'... Default is defined in oc_config.json->project->default_cluster. You should be logged in into corresponding cluster If cluster label in url should be different then cluster name, use '-A' flag to set custom cluster url label -C config file, default: 'oc_config.json' -d instanceName(s), deletes instances from logged in project e.g.: '-d test', see -i documentation -h help, prints this help -H host, used in default route url generation -i instance name(s) e.g.: installs given instances e.g.: 'test;prod' -k use(install/delete/validate/upgrade/generate yamls/backup) only specific Openshift/Kubernetes resource kind(s) e.g.: -k 'DeploymentConfig;Service;Route' Default is defined in oc_config.json->project->oc_resource_kinds -m component directory names e.g.: 'init-project;init-api;init-gateway;solr;api;gateway;public-ui;admin-ui;swagger;cron-jobs' omit flag or use '-m all' to select all defined in oc_config.json->project->component_dirs -n openshift project namespace e.g.: 'myNameSpace' if not set, current oc project name will be used as default namespace -N project name, used in default route url generation -o 'init', all directories which name string includes 'init' will be omitted, useful if you like to preserve data volumes during install/delete operations 'oc', only yaml files will be generated -p url prefix, adds prefix to all route urls, useful when running in Openshift sandbox in oder to avoid network routes conflicts -r openshift cluster IP range(first three numbers) e.g.: '112.20.14', last number will be randomly generated -s use this flag to change secrets directory. Default is 'secrets' -S 'numeric' or 'alphabetic', dirs and files sort type(relevant for running order), default is 'numeric' -t set custom 'templates_tt' directory -T directoryName, convert '.yaml' files extension into '.tt' extention inside of given directory -u instanceName(s), runs validation, creates 'validation_report.txt' und then runs upgrade of components that are modified, e.g.: -u test -v instanceName(s), validates given instance(s) between template version and Openshift version in cloud e.g.: -v test, report is written in 'validation_report.txt' file -y use(install/delete/validate/upgrade/generate yamls) only for yaml files that includes given substring -Y set custom 'templates_yaml' directory
see '-a multipleClusters' flag
oc_config.json magic nodes:
'instance_specific_data': instance will be automatically selected, e.g.: for json node 'instance_specific_data.api.test.limits.memory'
you can access instance specific data in tt template by
[% oc_config.instance_specific_data.api.limits.memory %] if current instance is 'test'
'instance_specific_name': '-instanceName' will be added at the end of each value, e.g.: if you have json node 'instance_specific_name.api'
with value 'my-api' then by accessing [% oc_config.instance_specific_name.api %] in tt template, in yaml file will be
written 'my-api-test' if current instance is 'test
see more examples in git: https://gitlab.com/code7143615/octoolkit/-/tree/master
Examples:
# install 'dev' and 'test' instances in 'clusterIntern' cluster
# you shoud be logged in in same cluster that you specified in '-c' flag
ocToolkit -c clusterIntern -i 'dev;test'
# deletes 'api' and 'solr' components on 'test' instance in currently logged in project
ocToolkit -d test -m 'api;solr'
# backups all components on 'test' instance in logged in project and remove clutter yaml nodes
ocToolkit -b test -a removeClutter
# validates 'api' and 'solr' components on 'test' instance in logged in project
# you shoud be logged in in same cluster that you specified in '-c' flag
ocToolkit -c clusterIntern -v test -m 'api;solr'
# upgrades 'solr' component on 'test' instance in logged in project but all dirs that have 'init' string in its name will me omitted
# you shoud be logged in in same cluster that you specified in '-c' flag
ocToolkit -c clusterIntern -u test -m solr -o init
# generate yaml templates(no installing) for oc resourse kinds 'DeploymentConfig' and 'Service'
# of 'solr' component for 'test' instance for 'clusterPublic' cluster
ocToolkit -c clusterPublic -i test -m solr -k 'DeploymentConfig;Service' -o oc
Place instance specific secretes in e.g.: secrets/instance/test/my_secret.txt for 'test' instance and
access them by [% secrets.instance_specific.item('my_secret.txt') %] from tt template
Make sure that oc_config.json->project node values are set correctly.
";sub addFlagValuesToConfig($); sub componentIsAllowed($$$$); sub generateUrl($$$$$$); sub removeClutter($$); sub _loopInstances($$);
my $clusterBaseAddress; my $advanceFeatures; my $backupSpecificInstances; my $cluster = "clusterPublic"; # default my $ocConfigFile; my $deleteInstances; my $host; my $installInstances; my $ocResourceKinds; my $componentDirs; my $namespace; my $projectName; # used in default url generation my $omit = ""; my $urlPrefix; my $clusterIpRange; my $secretsDir; my $sortType; my $templatesTTDir; my $yamlToTTconvertDir; my $upgradeInstances; my $validateInstances; my $specificYamlFile; my $templatesYamlDir;
my %opts; getopts("a:A:b:c:d:H:h:i:k:m:M:n:N:o:O:p:r:s:S:t:T:u:v:y:Y:", \%opts);
if($opts{h}){ print $help; exit; }
$clusterBaseAddress = $opts{A} if defined $opts{A}; $advanceFeatures = $opts{a} if defined $opts{a}; $backupSpecificInstances = $opts{b} if defined $opts{b}; $cluster = $opts{c} if defined $opts{c}; $ocConfigFile = $opts{C} if defined $opts{C}; $deleteInstances = $opts{d} if defined $opts{d}; $host = $opts{H} if defined $opts{H}; $installInstances = $opts{i} if defined $opts{i}; $ocResourceKinds = $opts{k} if defined $opts{k}; $componentDirs = $opts{m} if (defined $opts{m}) && ($opts{m} ne "all"); $namespace = $opts{n} if defined $opts{n}; $projectName = $opts{N} if defined $opts{N}; $omit = $opts{o} if defined $opts{o}; $urlPrefix = $opts{p} if defined $opts{p}; $clusterIpRange = $opts{r} if defined $opts{r}; $secretsDir = $opts{s} if defined $opts{s}; $sortType = $opts{S} if defined $opts{S}; $templatesTTDir = $opts{t} if defined $opts{t}; $yamlToTTconvertDir = $opts{T} if defined $opts{T}; $upgradeInstances = $opts{u} if defined $opts{u}; $validateInstances = $opts{v} if defined $opts{v}; $specificYamlFile = $opts{y} if defined $opts{y}; $templatesYamlDir = $opts{Y} if defined $opts{Y};
if(defined $installInstances){ die("Please set working cluster with -c flag, e.g.: perl ocToolkit.pl -c clusterPublic -i prod") if not defined $cluster; }
my $flag; $flag = $deleteInstances if defined $deleteInstances; $flag = $installInstances if defined $installInstances; $flag = $backupSpecificInstances if defined $backupSpecificInstances; $flag = $validateInstances if defined $validateInstances; $flag = $upgradeInstances if defined $upgradeInstances; die("Flags -i -d -b -v -u can't be left empty.") if (defined $flag) && (length($flag) eq 2) && ($flag =~ /\-/ );
if(not defined $clusterBaseAddress){ my $clusterLc = lc $cluster; $clusterBaseAddress = "apps.$clusterLc"; }
if(defined $deleteInstances){ my $myComponentDirs = $componentDirs; if($omit =~ /init/ && (defined $componentDirs)){ # remove init dirs my @componentsDirArray = split(';', $componentDirs); @componentsDirArray = (grep {$_ !~ /init/} @componentsDirArray); $myComponentDirs = join( ';', @componentsDirArray); } $myComponentDirs = "all" if not defined $myComponentDirs; print qx/oc project/; print "Deleting $myComponentDirs component(s) in '$deleteInstances' instance in '$cluster' cluster. \nPress enter to continue or ctrl+c to abort"; my $continue = <>; }
############################################################################ # install/delete/validate/update/backup/create yamls for specific instance # ############################################################################
# set 'multipleClusters' as default if(not defined $advanceFeatures){ $advanceFeatures = "multipleClusters"; }else{ $advanceFeatures .= ";multipleClusters"; } my $ocObj = OcToolkit->new( advanceFeatures => $advanceFeatures, clusterBaseAddress => $clusterBaseAddress, cluster => $cluster, ocConfigFile => $ocConfigFile, host => $host, ocResourceKinds => $ocResourceKinds, componentDirs => $componentDirs, namespace => $namespace, projectName => $projectName, omit => $omit, urlPrefix => $urlPrefix, clusterIpRange => $clusterIpRange, secretsDir => $secretsDir, sortType => $sortType, templatesTTDir => $templatesTTDir, yamlToTTconvertDir => $yamlToTTconvertDir, specificYamlFile => $specificYamlFile, templatesYamlDir => $templatesYamlDir, addFlagValuesToConfig => \&addFlagValuesToConfig, componentIsAllowed => \&componentIsAllowed, generateUrl => \&generateUrl, removeClutter => \&removeClutter, removeClutterBackup => \&removeClutterBackup);
# $ocObj->setParams({omit => "oc"}); _loopInstances($deleteInstances, "delete") if defined $deleteInstances; _loopInstances($installInstances, "install") if defined $installInstances; _loopInstances($upgradeInstances, "upgrade") if defined $upgradeInstances; _loopInstances($validateInstances, "validate") if defined $validateInstances; if(defined $backupSpecificInstances){ if($backupSpecificInstances eq "all"){ $ocObj->backupWholeOCProject(); }else{ _loopInstances($backupSpecificInstances, "backup"); } } $ocObj->convertYamlToTTExtention($yamlToTTconvertDir) if defined $yamlToTTconvertDir;
##################################################################################### # use this functions to add custom config/logic without need to change OcToolkit.pm # #####################################################################################
# add some script input flag value to config file, access them then in TT Template by [% my_custom_value %] sub addFlagValuesToConfig($){ my ($config) = @_;
$config->{my_custom_value} = "some custom value received from flag";
return $config;
}# some specific rules about what components are allowed to be installed on given cluster and instance # if you need completely different tt template files for specific clusters use '-a multipleClusters' flag # put tt template file into nested directory, named by your cluster, inside of your tt template component dirs sub componentIsAllowed($$$$){ my ($myTemplateName, $myDir, $myCluster, $myInstance) = @_;
# my $clusterLowerCase = lc $myCluster; # if($clusterLowerCase =~ /clusterPublic/){ # if($myTemplateName =~ /route/){ # return 0 if $myDir =~ /solr/; # return 0 if $myDir =~ /api/; # return 0 if $myDir =~ /admin\-ui/; # } # return 0 if $myDir =~ /swagger/; # }
return 1;
}# define default routes url pattern sub generateUrl($$$$$$){ my ($urlPrefix, $projectName, $componentName, $instanceKey, $clusterBaseAddress, $host) = @_;
if(defined $urlPrefix){
$urlPrefix .= "-";
}else{
$urlPrefix = "";
}
if(defined $projectName){
$projectName .= "-";
}else{
$projectName = "";
}
if(defined $clusterBaseAddress){
$clusterBaseAddress .= ".";
}else{
$clusterBaseAddress = "";
}
return $urlPrefix.$projectName.$componentName."-".$instanceKey.".".$clusterBaseAddress.$host;
}# ignore some specific fields during validation and upgrade sub removeClutter($$){ my ($ocJsonHash, $params) = @_;
$ocJsonHash = removeClutterBackup($ocJsonHash, $params);
my $dir = $params->{dir};
my $templateName = $params->{templateName};
my $ocKind = $params->{ocKind};
my $ocName = $params->{ocName};
# some extra fields(in comparison to backup) to be ignored during validation and upgrade
foreach my $i ((0..7)){
if($ocKind eq "BuildConfig"){
if((defined $ocJsonHash->{spec}) &&
(defined $ocJsonHash->{spec}->{triggers}) &&
(defined $ocJsonHash->{spec}->{triggers}->[$i]) &&
(defined $ocJsonHash->{spec}->{triggers}->[$i]->{github}) &&
(defined $ocJsonHash->{spec}->{triggers}->[$i]->{github}->{secret})
){
delete $ocJsonHash->{spec}->{triggers}->[$i]->{github}->{secret};
}
if((defined $ocJsonHash->{spec}) &&
(defined $ocJsonHash->{spec}->{triggers}) &&
(defined $ocJsonHash->{spec}->{triggers}->[$i]) &&
(defined $ocJsonHash->{spec}->{triggers}->[$i]->{generic}) &&
(defined $ocJsonHash->{spec}->{triggers}->[$i]->{generic}->{secret})
){
delete $ocJsonHash->{spec}->{triggers}->[$i]->{generic}->{secret};
}
}
if($ocKind eq "ImageStream"){
if((defined $ocJsonHash->{spec}) &&
(defined $ocJsonHash->{spec}->{tags}) &&
(defined $ocJsonHash->{spec}->{tags}->[$i]) &&
(defined $ocJsonHash->{spec}->{tags}->[$i]->{importPolicy})
){
delete $ocJsonHash->{spec}->{tags}->[$i]->{importPolicy};
}
}
if($ocKind eq "DeploymentConfig"){
if((defined $ocJsonHash->{spec}) &&
(defined $ocJsonHash->{spec}->{template}) &&
(defined $ocJsonHash->{spec}->{template}->{spec}) &&
(defined $ocJsonHash->{spec}->{template}->{spec}->{containers}) &&
(defined $ocJsonHash->{spec}->{template}->{spec}->{containers}->[$i]) &&
(defined $ocJsonHash->{spec}->{template}->{spec}->{containers}->[$i]->{resources}) &&
(not defined $ocJsonHash->{spec}->{template}->{spec}->{containers}->[$i]->{resources}->{limits}) &&
(not defined $ocJsonHash->{spec}->{template}->{spec}->{containers}->[$i]->{resources}->{requests})
){
$ocJsonHash->{spec}->{template}->{spec}->{containers}->[$i]->{resources} = {};
}
}
}
delete $ocJsonHash->{spec}->{clusterIPs};
delete $ocJsonHash->{spec}->{clusterIP};
delete $ocJsonHash->{spec}->{volumeName} if $ocKind eq "PersistentVolumeClaim";
return $ocJsonHash;
}# ignore some specific fields during backup of the whole project sub removeClutterBackup($$){ my ($ocJsonHash, $params) = @_;
my $dir = $params->{dir};
my $templateName = $params->{templateName};
my $ocKind = $params->{ocKind};
my $ocName = $params->{ocName};
delete $ocJsonHash->{status};
delete $ocJsonHash->{metadata}->{annotations}->{'openshift.io/generated-by'};
delete $ocJsonHash->{metadata}->{annotations}->{'openshift.io/restore-server-version'};
delete $ocJsonHash->{metadata}->{annotations}->{'openshift.io/backup-server-version'};
delete $ocJsonHash->{metadata}->{annotations}->{'openshift.io/backup-registry-hostname'};
delete $ocJsonHash->{metadata}->{annotations}->{'openshift.io/migration-registry'};
delete $ocJsonHash->{metadata}->{annotations}->{'openshift.io/restore-registry-hostname'};
delete $ocJsonHash->{metadata}->{annotations}->{'openshift.io/image.dockerRepositoryCheck'};
delete $ocJsonHash->{metadata}->{annotations}->{'kubectl.kubernetes.io/last-applied-configuration'};
delete $ocJsonHash->{metadata}->{annotations}->{'kubernetes.io/service-account.name'};
delete $ocJsonHash->{metadata}->{annotations}->{'kubernetes.io/service-account.uid'};
delete $ocJsonHash->{metadata}->{annotations}->{'openshift.io/token-secret.value'};
delete $ocJsonHash->{metadata}->{annotations}->{'openshift.io/token-secret.name'};
delete $ocJsonHash->{metadata}->{annotations}->{'kubernetes.io/created-by'};
delete $ocJsonHash->{metadata}->{annotations}->{'volume.beta.kubernetes.io/storage-provisioner'};
delete $ocJsonHash->{metadata}->{annotations}->{'pv.kubernetes.io/bind-completed'};
delete $ocJsonHash->{metadata}->{annotations}->{'pv.kubernetes.io/bound-by-controller'};
delete $ocJsonHash->{metadata}->{annotations}->{'volume.kubernetes.io/storage-provisioner'};
delete $ocJsonHash->{metadata}->{labels}->{'migration.openshift.io/migrated-by-migmigration'};
delete $ocJsonHash->{metadata}->{labels}->{'migration.openshift.io/migrated-by-migplan'};
delete $ocJsonHash->{metadata}->{labels}->{'velero.io/backup-name'};
delete $ocJsonHash->{metadata}->{labels}->{'velero.io/restore-name'};
delete $ocJsonHash->{metadata}->{labels}->{'app.kubernetes.io/component'};
delete $ocJsonHash->{metadata}->{labels}->{'app.kubernetes.io/instance'};
delete $ocJsonHash->{metadata}->{resourceVersion};
delete $ocJsonHash->{metadata}->{uid};
delete $ocJsonHash->{metadata}->{creationTimestamp};
delete $ocJsonHash->{metadata}->{generation};
delete $ocJsonHash->{metadata}->{managedFields};
delete $ocJsonHash->{metadata}->{selfLink};
delete $ocJsonHash->{metadata}->{deletionTimestamp};
delete $ocJsonHash->{metadata}->{deletionGracePeriodSeconds};
foreach my $i ((0..7)){
if((defined $ocJsonHash->{spec}) &&
(defined $ocJsonHash->{spec}->{triggers}) &&
(defined $ocJsonHash->{spec}->{triggers}->[$i]) &&
(defined $ocJsonHash->{spec}->{triggers}->[$i]->{imageChangeParams}) &&
(defined $ocJsonHash->{spec}->{triggers}->[$i]->{imageChangeParams}->{lastTriggeredImage})
){
delete $ocJsonHash->{spec}->{triggers}->[$i]->{imageChangeParams}->{lastTriggeredImage};
}
if((defined $ocJsonHash->{spec}) &&
(defined $ocJsonHash->{spec}->{tags}) &&
(defined $ocJsonHash->{spec}->{tags}->[$i]) &&
(defined $ocJsonHash->{spec}->{tags}->[$i]->{generation})
){
delete $ocJsonHash->{spec}->{tags}->[$i]->{generation};
}
if((defined $ocJsonHash->{spec}) &&
(defined $ocJsonHash->{spec}->{template}) &&
(defined $ocJsonHash->{spec}->{template}->{spec}) &&
(defined $ocJsonHash->{spec}->{template}->{spec}->{containers}) &&
(defined $ocJsonHash->{spec}->{template}->{spec}->{containers}->[$i]) &&
(defined $ocJsonHash->{spec}->{template}->{spec}->{containers}->[$i]->{image})
){
delete $ocJsonHash->{spec}->{template}->{spec}->{containers}->[$i]->{image};
}
}
return $ocJsonHash;
}sub _loopInstances($$){ my ($instancesString, $methodName) = @_;
my @instances = split(';', $instancesString);
foreach my $instance (@instances){
my $methodNameU = ucfirst $methodName;
print "$methodNameU instance: $instance\n";
$ocObj->$methodName($instance) if $ocObj->can($methodName);
}
}1;