# # osds_acfsremote.pm # # Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. # # NAME # osds_acfsremote.pm - # # DESCRIPTION # # # NOTES # # # use strict; package osds_acfsremote; use English; use File::Copy; use acfslib; use osds_acfslib; use osds_unix_linux_acfslib; use osds_acfsroot; use osds_acfsrVerifyConfig; use POSIX qw(strftime); use Socket; our @ISA = qw(Exporter); our @EXPORT = qw( osds_acfsr_transport_config osds_acfsr_transport_list osds_acfsr_transport_change_add_acl osds_acfsr_transport_change_remove_acl osds_acfsr_transport_change_setup_target osds_acfsr_transport_change_delete_target osds_acfsr_supported osds_acfsr_installed osds_acfsr_loaded $log_path ); our $log_path = get_log_path(); my $d = 3; my $log_fh; my $log_valid = 0; my $iqn_pre="iqn.2015-12.com.oracle\\:acfsr"; my ($UNAME_R) = `uname -r`; chomp ($UNAME_R); # Transport rescan # # 1. Description # # This functionality will sync up the logged in sessions with the # available targets from the DC node. # # 2. Workflow # # Get existing logged in acfs remote iSCSI targets using # # 'iscsiadm -m session' # # Get available targets using iSCSI scan command. # # FOR EACH logged in session # Scan available target list # if NOT FOUND # Logout of session # # For EACH available target # Scan session targets # if NOT FOUND # Login to target using iSCSI login command sub osds_acfsr_transport_rescan { # Will look like nshga2601.iSCSI.0 my $current_transport = shift; # Strip away the hostname my $symlink_device = ""; if($current_transport =~ /\w+\.(\w+\.\w+)/) { $symlink_device = $1; } # Will be an IP address my $DSC_IP = shift; my ($rc,@out); lib_trace(9999,"Enter transport_config --rescan"); my ($logged_rc,@logged_entries); my $logged_entry; my ($discovered_rc,@discovered_entries); my $discovered_entry; lib_trace(9999,"Retrieve logged in sessions"); ($logged_rc,@logged_entries) = execute_command('/sbin/iscsiadm -m session'); if($logged_rc > 0) { lib_trace(9999,"Error retrieving logged in sessions"); } # Logged entries will look like # tcp: [4] 10.137.12.155:3260,1 iqn.2015-12.com.oracle:acfsr:000000:1:vol-000 # (non-flash) lib_trace(9999,"Retrieve discovered devices in Domain Services " . "Cluster @ $DSC_IP"); ($discovered_rc,@discovered_entries) = execute_command("/sbin/iscsiadm -m discovery " . "-t st -p $DSC_IP "); # Discovered entries will look like # 10.137.12.155:3260,1 iqn.2015-12.com.oracle:acfsr:000000:1:vol-000 if($discovered_rc > 0) { lib_trace(9999,"Error retrieving available transports in $DSC_IP"); $rc = 1; goto done; } if(-e "/dev/acfsr/$symlink_device") { my @devices; # translate $current_transport into an entry format to find a match # Use find to get the devices under a given transport (in # $symlink_device). Readlink is then used to resolve the symlink # we expect to find to the actual device under /dev/sd*. ($rc, @devices) = execute_command ("/bin/find -P /dev/acfsr/$symlink_device -maxdepth 1 -type l -exec /bin/readlink -f {} \\;"); # @devices should contain entries like: # /dev/sda foreach (@devices) { my $current_device = $_; # Strip the /dev/ part $current_device =~ s/\/dev\///; chomp($current_device); ($rc,@out) = execute_command("ls /sys/block/$current_device/device/scsi_device/"); # the output should be a single bus entry # 9:0:0:0 if($rc > 0) { lib_trace(9999,"Failed to list the bus entry for " . "$current_device"); goto done; } # Use acfsr_member to get device iqn. chomp($out[0]); ($rc, @out) = execute_command("/usr/lib/udev/acfsr_member " . "$out[0] -I"); my $iqn = $out[0]; # $out[0] should contain something like # iqn.2015-12.com.oracle:acfsr:000000:1:vol-000 # Log out any entries that may have become stale if they are not # found in the DSC portal discovered entries. for $logged_entry (@logged_entries) { my $logged_entry_short; my @logged_entry_split = split ' ',$logged_entry; $logged_entry_short = $logged_entry_split[2] . " " . $logged_entry_split[3]; # A logged entry should match a discovered entry # $logged_entry should look like: # tcp: [4] 10.137.12.155:3260,1 iqn.2015-12.com.oracle:acfsr:000000:1:vol-000 (non-flash) # $logged_entry_short should look like # 10.137.12.155:3260,1 iqn.2015-12.com.oracle:acfsr:000000:1:vol-000 # which will then match the format of discovered_entries unless (grep { /$logged_entry_short/ } @discovered_entries) { my $target; if($logged_entry =~ /.*($iqn).*/) { $target = $1; # log out lib_trace(9999,"Log out $target"); ($rc,@out) = execute_command("iscsiadm -m node -T " . "$iqn -u"); } } } } ($discovered_rc,@discovered_entries) = execute_command("/sbin/iscsiadm -m discovery " . "-t st -p $DSC_IP "); for $discovered_entry (@discovered_entries) { lib_trace(9999,"Looking for $discovered_entry in " . "@logged_entries"); unless(grep { /$discovered_entry/ } @logged_entries) { my $target; if($discovered_entry =~ / ($iqn_pre\S+)/) { $target = $1; ($rc, @out) = execute_command("iscsiadm -m node " . "-p $DSC_IP " . " -T $target --rescan"); lib_trace(9999,"Log in $target"); ($rc, @out) = execute_command("iscsiadm -m node " . "-p $DSC_IP " . " -T $target -l"); lib_trace(9999,"rc = $rc"); lib_trace(9999,"Stop $target from auto login on boot"); ($rc, @out) = execute_command("iscsiadm -m node " . " -T $target " . " -o update " . " -n node.startup " . " -v manual" ); lib_trace(9999,"rc = $rc"); } } } $rc = 0; } else { lib_trace(9999,"No sessions to operate on."); $rc = 0; } done: lib_trace(9999,"Exit transport_config --rescan"); return $rc; } # Transport refresh # # 1. Description # # This functionality will logout of any existing ACFS remote sessions # and attempt to login to an available iSCSI targets. One area where # this will used is if a MC node needs to reconnect to the VIP because # of DC node failover. This is a reinitilization of the entire Transport # Session - every underlying ISCSI session associated with it will be # paused and affected. # # 2. Workflow # # For each acfs remote session # logout all sessions for a given transport # # Run iSCSI scan command # # For each acfs remote session target returned by iscsiadm discovery # for the provided transport # login with iSCSI login command manually # sub osds_acfsr_transport_refresh { lib_trace(9999,"Enter transport_config --refresh"); my $current_transport = shift; # Strip away the hostname my $symlink_device = ""; if($current_transport =~ /\w+\.(\w+\.\w+)/) { $symlink_device = $1; } my $DSC_IP = shift; my ($rc,@out); my $output_entry; my ($logged_rc,@logged_entries); my $logged_entry; my @devices; my ($discovered_rc,@discovered_entries); my $discovered_entry; lib_trace(9999,"Retrieve logged in sessions"); ($logged_rc,@logged_entries) = execute_command("/sbin/iscsiadm -m session ". "| grep $DSC_IP"); # Logged entries will look like # tcp: [4] 10.137.12.155:3260,1 iqn.2015-12.com.oracle:acfsr:000000:1:vol-000 # (non-flash) if($logged_rc > 0) { lib_trace(9999,"Error retrieving logged in sessions"); } lib_trace(9999,"Retrieve discovered devices in Domain Services " . "Cluster @ $DSC_IP"); ($discovered_rc,@discovered_entries) = execute_command("/sbin/iscsiadm -m " . "discovery -t st -p $DSC_IP " . "| grep $DSC_IP"); # Discovered entries will look like # 10.137.12.155:3260,1 iqn.2015-12.com.oracle:acfsr:000000:1:vol-000 if($discovered_rc > 0) { lib_trace(9999,"Error retrieving available transports in $DSC_IP"); $rc = 1; goto done; } # translate $current_transport into an entry format to find a match if(-e "/dev/acfsr/$symlink_device") { ($rc, @devices) = execute_command ("/bin/find -P /dev/acfsr/$symlink_device -maxdepth 1 -type l -exec /bin/readlink -f {} \\;"); # @devices should contain entries like: # /dev/sda foreach (@devices) { my $current_device = $_; # Strip the /dev/ part $current_device =~ s/\/dev\///; chomp($current_device); ($rc,@out) = execute_command("ls /sys/block/$current_device/device/scsi_device/"); # the output should be a single bus entry # 9:0:0:0 if($rc > 0) { lib_trace(9999,"Failed to list the bus entry for " . "$current_device"); goto done; } # Use acfsr_member to get device iqn. chomp($out[0]); ($rc, @out) = execute_command("/usr/lib/udev/acfsr_member $out[0] -I"); # $out[0] should contain something like # iqn.2015-12.com.oracle:acfsr:000000:1:vol-000 my $iqn = $out[0]; # Log out any entries that may have become stale if not found # in the DSC portal discovered entries. for $output_entry (@logged_entries) { my $target; if($output_entry =~ /.*($iqn).*/) { $target = $1; # log out lib_trace(9999,"Log out $target"); ($rc,@out) = execute_command("iscsiadm -m node -T " . "$target -u"); lib_trace(9999,"rc = $rc"); } } lib_trace(9862,"Scanning iSCSI devices in Domain Services " . "Cluster %s ","$DSC_IP"); } } else { # No active sessions. Just login. lib_trace(9999,"No active sessions found, login discovered entries"); } # This is the list of all discovered iSCSI entries found in the DSC # for the provided DSC_IP (transport IP) # The sessions are logged manually to prevent # other MCs from getting their exports incorrectly included, thus # providing a granular refresh limited to just this DSC_IP for $output_entry (@discovered_entries) { lib_trace(9999, $output_entry); my $iqn; my $target; if($output_entry =~ /.*(iqn.*)/) { $target = $1; ($rc, @out) = execute_command ("iscsiadm -m node " . "-p $DSC_IP " . " -T $target --rescan"); $iqn = $1; lib_trace(9863,"Logging in %s", $iqn); my $command = "iscsiadm -m node -p $DSC_IP -T $iqn -l "; lib_trace(9999, "Executing '$command'"); ($rc,@out) = execute_command($command); lib_trace(9999,"return code = $rc"); lib_trace(9999,"Stop $iqn from auto login on boot"); ($rc, @out) = execute_command("iscsiadm -m node " . " -T $iqn " . " -o update " . " -n node.startup " . " -v manual" ); lib_trace(9999,"rc = $rc"); } } done: lib_trace(9999,"Exit transport_config --refresh"); return $rc; } # This function will perform the transport configuration for acfs remote. # This function gets called by 'acfsroot transport_config'. # This function expects 2 arguments: # 1) 'refresh' or 'rescan' # 2) comma-separated list of transports to which apply the operation # # iSCSI scan # # On Linux, the following command should be used to do the scan. # # iscsiadm -m discovery -t st -p # # The above command will populate the database of available targets on # the node. It will also return the available targets in the output. # # iSCSI login # # On Linux, the following command should be used to do the login for # each target. # # iscsiadm -m node -p -T -l # # The iscsi initiator allows multiple connections per target, so it is # important NOT to login to the same target more than once. If so, # multiple SCSI devices will show up that point to the same target on # the DCN. # # iSCSI logout # # On Linux, the following command should be used to logout a target. # # iscsiadm -m node -T -u # # iSCSI session # # On Linux, the following command should be used to retrieve the list # of active iSCSI sessions. # # iscsiadm -m session # # We need to further filter/verify this so we only operate on iSCSI # sessions related to ACFS Remote # sub osds_acfsr_transport_config { my $operation = shift; my $transport_string = shift; # this is a comma separated transport list # nshga2601.iSCSI.0,nshga2601.iSCSI.2 my @transport_list; my @ip_list; my $seqnum = 0; my $transport_type = "iSCSI"; # Default to iSCSI my $ignore_entry = 1; my $match_found = 0; my $name = ""; my $op_applied = 0; my $ret = USM_FAIL; # Assume fail if the corresponding # transport operation is not even reached. lib_trace( 9176, "Entering '%s'", "osds_acfsr_transport_config"); # split into an array my @transports_to_config = split /,/,$transport_string; my $rc; my @advmutil_output; foreach (@transports_to_config) { my $current_transport = $_; if($current_transport !~ /\w+\.BLK|[iI]SCSI\.\d+/) { lib_trace(9999,"Invalid transport format: $current_transport"); return 1; } ($transport_type, $seqnum) = $current_transport =~ /\w+\.(\w+)\.(\d+)/; # We want verbose to get the xml data output, which will (hopefully) # contain the DSC's address. ($rc,@advmutil_output) = execute_command("$ADVMUTIL session list -v " . "-i $transport_type.$seqnum "); # Sample output: #[root@nshgc1712 ~]# /sbin/advmutil session list -v -i ISCSI.0 #nodeName transportType sequenceNum state # (XML data) #--------------------------------------------------------- #nshgc1712 iSCSI 0 ONLINE # # ISCSI # # # cluster99_ISCSI_0 # nshgc171112-havip1 # 1 # # # # BEGIN XML PARSE # This XML parsing is not being done via a library, it should be. # Allan Graves did tell me to use the library once this was on code review # but I'm not going to do it in this transaction due to time constraints. foreach my $line (@advmutil_output) { # Look for # if ($line =~ /(\w+)\W+(BLK|[iI]SCSI)\W+(\d+).*/) { # Match none or some non-word # then a word (iSCSI or BLK) # then at least one non word # then one or more digit (seqnum) # and then put it all together my $session_list_element = uc ("$1.$2.$3"); # Now, loop the passed input to find matches if( uc($current_transport) =~ $session_list_element) { # Match found! $match_found = 1; lib_trace(9999,"Applying '$operation' to " . "'$current_transport'"); $op_applied = 0; # We haven't done anything to this match } # We looped all transports input, did we find a match? if($match_found == 1) { # Set this so the XML data following won't be ignored $ignore_entry = 0; # Reset for next transport in the list $match_found = 0; } else { # No need to process the XML data $ignore_entry = 1; } } elsif($ignore_entry == 0 && $line !~ /^-+/) { my $in_IP_tag = 0; my $xml_DSC_IP = ""; $line =~ s/\s//g; # If the line is not the separator being printed, we should be # at the XML data. Now, look for an IP address. if(($line =~ /.*.*/ || $line =~ /.*.*/) || $in_IP_tag == 1 && $op_applied == 0) { $in_IP_tag = 1; # For a DNS name I would look for word[.word]* format my $name_regex = ">(.+)<"; # Maybe the ip/name is in the same line, so the tags may # be present in $line if($line =~ /\D*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).*/) { # Look for zero or more non-digits, then # look for one to three digits followed by a period # 3 times and one last set of 1-3 digits. # OR # Look for the xml tags and brackets and grab the name # OR # Look for a name $xml_DSC_IP = $1; # The problem with this is if a transport has more than # one IP address (as seen in the sample provided by Allan) # I would use the last one. He told me not to worry about # it. } elsif ($line =~ /$name_regex/) { $name = $1; my @addresses = gethostbyname($name); # filter out other data returned by gethostbyname @addresses = map { inet_ntoa($_) } @addresses[4 .. $#addresses]; $xml_DSC_IP = shift @addresses; } # At this point we should have an IP address if($op_applied == 0 && $xml_DSC_IP ne '') { # Clear out any node information of stale targets execute_command("/sbin/iscsiadm -m discovery " . "-t st -p $xml_DSC_IP -o delete"); lib_trace(9999, "Using '$xml_DSC_IP' for transport " . "'$current_transport'"); if ($operation eq 'rescan') { $ret = osds_acfsr_transport_rescan($current_transport, $xml_DSC_IP); } elsif ($operation eq 'refresh') { $ret = osds_acfsr_transport_refresh($current_transport, $xml_DSC_IP); } else { # Unknown operation, should never get here $ret = USM_FAIL; } $op_applied = 1; } else { lib_trace(9999,"Invalid transport configuration, no host" . " IP address or hostname is present. Verify ". "configuration for $current_transport"); } } if(($line =~ /.*.*/ || $line =~ /.*.*/) && $in_IP_tag == 0 && $op_applied == 0) { $in_IP_tag = 1; } # Check if we're leaving the xml tag if($in_IP_tag == 1 && ($line =~ /.*<\/BINDADDR>.*/ || $line =~ /.*<\/BINDNAME>.*/)) { $in_IP_tag = 0; } } } } # END XML PARSE # This XML parsing is not being done via a library, it should be. # Allan Graves did tell me to use the library once this was on code review # but I'm not going to do it in this transaction due to time constraints. lib_trace(9999,"Exit osds_acfsr_transport_config"); return $ret; } # This function will list the available transports and their status. # This function gets called by 'acfsroot transport_list'. # The output gets parsed by the crs acfs agent. # # V1 - List the transport IDs found in # iscsi: /etc/iscsi/initiatorname.iscsi # blk: /sys/hypervisor/uuid # Mark BLK as not in use. sub osds_acfsr_transport_list { my $iscsi_path = "/etc/iscsi/initiatorname.iscsi"; my $blk_path = "/sys/hypervisor/uuid"; my $iscsi_identifier = ""; my $blk_identifier = ""; my $line; open(FH,"<$iscsi_path") || die "cannot open $iscsi_path for read: $!"; while ($line = ) { if($iscsi_identifier eq "") { $iscsi_identifier = $line; chomp($iscsi_identifier); $iscsi_identifier =~ s/InitiatorName=//g; } else { return USM_FAIL; # This means there are multiple lines in the identifier file. # Shouldn't happen. } } close FH; #Output of the acfsroot transport_list looks like: # $ acfsroot transport_list # ISCSI: INUSE : IQN:foo.123 # BLK: NOTINUSE : 1111-XXXX # print "ISCSI: " ; print $iscsi_identifier ne "" ? "INUSE":"NOTINUSE"; print $iscsi_identifier ne "" ? " : $iscsi_identifier \n" : "\n"; print "BLK: " ; print $blk_identifier ne "" ? "INUSE":"NOTINUSE"; print $blk_identifier ne "" ? " : $blk_identifier \n" : ":\n"; return USM_SUCCESS; } sub execute_command { my $command = join ' ', @_; my @output; my $output_string; my $rc; my $timestamp = strftime "%F %T", localtime; lib_trace(9999, "Executing - $command"); @output = qx{$command 2>>$log_path}; $rc = $? >> 8; chomp(@output); $output_string = join("\n",@output); lib_trace(9999, $output_string); if(open(my $fh, '>>', $log_path)) { print $fh "[$$] [$timestamp] "; print $fh "Executing - $command\n"; print $fh "$output_string"; close $fh; } else { lib_trace(9999, "unable to write to $log_path"); } return ($rc, @output); } # This function will modify the values of the acfslib::acfsr hash. # The first value is 'True' when ACFS Remote is supported, 'False' otherwise. # In order to determine if it is supported we # - Retrieve the passed argument. It will either be "DOMAIN SERVICES" # or "MEMBER". # - Depending on cluster class, determine if the current OS is supported. # If it is supported, other values will be pushed into the array # Is ISCSI supported? 'True' or 'False'. sub osds_acfsr_supported { my $cluster_class = shift; my $rc = 0; if($cluster_class eq 'MEMBER') { $acfslib::acfsr{'ACFS Remote'} = 'True'; $rc = 1; } elsif($cluster_class eq 'DOMAIN SERVICES') { my $kernel_name = `uname -s`; my $kernel_release = $UNAME_R; chomp($kernel_name); if($kernel_name eq 'Linux') { $kernel_release =~ s/(\d+\.\d+)\..*/$1/g; if($kernel_release ge '3.8') { $acfslib::acfsr{'ACFS Remote'} = 'True'; $rc = 1; } } } else { $acfslib::acfsr{'ACFS Remote'} = 'False'; } # Check ISCSI support # Allan said this is always supported. $acfslib::acfsr{'iSCSI'} = 'True'; # # Check if this is an ODA # if(acfslib::isODA()) # { # # Xen Blkfrnt/blkback support # $acfslib::acfsr{'xen blkfrnt/blkback'} = 'True'; # } return $rc; } # This function will modify the values of the acfslib::acfsr hash. # The first value is 'True' when ACFS Remote is installed, 'False' otherwise. # In order to determine if it is installed we need to look for # /etc/modprobe.d/oracleadvm.conf (Linux location, this may vary in other OS) # If found, read it and look for asm_acfsr_mode option # As of 2/3/16 modes are: # DOMAINSERVICES = 1 # MEMBER = 2 # The list can be found in acfsroot.pl. # Perhaps I should move that list somewhere else? # Any of those modes mean 'installed'. Any other value (or a lack of one) # means not installed. # If it is supported, other values will be pushed into the array # Is ISCSI setup? 'True' or 'False'. sub osds_acfsr_installed { my $cluster_class = shift; my $mode = ''; my $fh; my $line; my @clusterinfo; my @iscsi_inst; my $iscsi_rc; my $rc; ($rc, @clusterinfo) = execute_command("$ACFSUTIL cluster info"); for (@clusterinfo) { if(/ACFS Remote mode: \[(.*)\]/) { $mode = $1; } } if($cluster_class eq $mode) { $acfslib::acfsr{'ACFS Remote'} = 'True'; $rc = 1; } else { $acfslib::acfsr{'ACFS Remote'} = 'False'; $rc = 0; } # Determine if iscsi is installed. ($iscsi_rc, @iscsi_inst) = execute_command('/sbin/service iscsid status'); # This check might break if we run in a shell in another language if(grep (/unrecognized service/, @iscsi_inst)) { lib_trace(9999, "iSCSI service is not running"); $acfslib::acfsr{'iSCSI'} = 'False'; $rc *= 0; } else { $acfslib::acfsr{'iSCSI'} = 'True'; $rc *= 1; } # # We would check for xen blkfrnt/blkback support via isODA. # # For now, we aren't going to check/show that. # if(acfslib::isODA()) # { # # Xen Blkfrnt/blkback support # $acfslib::acfsr{''} = 'True'; # } # else # { # $acfslib::acfsr{''} = 'True'; # } return $rc; } # This function will modify the values of the acfslib::acfsr hash. # The first value is 'True' when ACFS Remote is loaded, 'False' otherwise. # Is ISCSI setup and running? 'True' or 'False'. sub osds_acfsr_loaded { my $cluster_class = shift; my $mode = ''; my $fh; my $line; my @clusterinfo; my @iscsi_inst; my $iscsi_rc; my $rc; ($rc, @clusterinfo) = execute_command("$ACFSUTIL cluster info"); for (@clusterinfo) { if(/ACFS Remote mode: \[(.*)\]/) { $mode = $1; } } if($cluster_class eq $mode) { $acfslib::acfsr{'ACFS Remote'} = 'True'; $rc = 1; } else { $acfslib::acfsr{'ACFS Remote'} = 'False'; $rc = 0; } # Determine if iscsi is installed. ($iscsi_rc, @iscsi_inst) = execute_command('/sbin/service iscsid status'); # This check might break if we run in a shell in another language if(grep (/running/ ,@iscsi_inst)) { $acfslib::acfsr{'iSCSI'} = 'True'; $rc *= 1; } else { $acfslib::acfsr{'iSCSI'} = 'False'; $rc *= 0; } # if(acfslib::isODA()) # { # # Xen Blkfrnt/blkback support # $acfslib::acfsr{''} = 'True'; # } # else # { # $acfslib::acfsr{''} = 'True'; # } return $rc; } # This function adds the specified nodeid to a DSF's ACL. # It also sets the cmdsn_depth value for the target and the nr_requests for the # virtual block device. sub osds_acfsr_transport_change_add_acl { my $dsf = shift; my $nodeid = shift; my $guid = shift; my $return_code = USM_SUCCESS; my ($rc, @output); my @transport_list; my $iqn; my $block_name; my $path = ""; my $attr; my $mcid; my $whoami = (caller(0))[3]; lib_trace(9999, "Enter $whoami"); # We have to be able to add the nodeid to ALL ACLs. When we have multiple # transports, the nodeid needs to get added to each one. For that purpose, # we need to get the transport list and loop through each one. if(defined($guid)) { $guid = "-g " . $guid; } else { $guid = ""; } ($rc,@transport_list) = execute_command("$ADVMUTIL transport list $guid |" . " grep 'iSCSI.*[0-9]\\{1,\\}' | uniq "); foreach my $elem (@transport_list) { my ($seqnum) = $elem =~ /iSCSI.*(\d+)/; $iqn = iscsi_get_iqn($seqnum,$dsf); $path = "/iscsi/$iqn/tpg1/acls/"; $return_code = run_targetcli($path, "create $nodeid"); if($return_code != USM_SUCCESS) { goto done; } $attr = "/sys/kernel/config/target/iscsi/$iqn/tpgt_1/" . "acls/$nodeid/cmdsn_depth"; ($rc,@output) = execute_command("echo 512 > $attr"); # Since this is all iscsi (for now), always use 01 as the transport type. $iqn =~ /.*vol-(\d\d\d)/; my $vol_number = $1; if(! defined $vol_number) { return USM_FAIL; } ($mcid) = $dsf =~ /\/dev\/acfsr\/(\d\d\d\d\d\d)-\d\d-\d\d/; my $nr_path = "/sys/devices/virtual/block/acfsr\\!$mcid-01-$vol_number" . "/queue/nr_requests"; ($rc,@output) = execute_command ("echo 1024 > $nr_path"); } done: # Test validity of /dev/acfsr and output of 'targetcli ls'. osds_acfsrVerifyConfig(); lib_trace(9999,"Exit $whoami with rc = $return_code"); return $return_code; } sub osds_acfsr_transport_change_remove_acl { my $dsf = shift; my $nodeid = shift; my $guid = shift; my $return_code = USM_SUCCESS; my $iqn; my $path = ""; my $whoami = (caller(0))[3]; my ($rc, @output); lib_trace(9999, "Enter $whoami"); if(defined($guid)) { $guid = "-g " . $guid; } else { $guid = ""; } ($rc,@output) = execute_command("$ADVMUTIL transport list $guid | " . "grep 'iSCSI.*[0-9]\{1,\}'"); for my $elem (@output) { my ($seqnum) = $elem =~ /iSCSI.*(\d+)/; $iqn = iscsi_get_iqn($seqnum,$dsf); if (! -d "/sys/kernel/config/target/iscsi/$iqn/tpgt_1/acls/$nodeid") { lib_trace(9999,"$nodeid does not exist for $iqn"); $return_code = USM_FAIL; } else { $path = "/iscsi/$iqn/tpg1/acls/"; $return_code = run_targetcli($path, "delete $nodeid"); } } return $return_code; } # # This function creates a new iscsi target using the specified DSF for the # specified remote cluster. # Input: # - $guid = GUID of the cluster for which the storage is going to be # exported. # - $dsf = Path to device special file being exported. # - $nodeid (optional) = specific nodeid to setup the target to. # Output: # Returns a counter of how many targetcli commands failed. 0 means # everything executed successfully. # sub osds_acfsr_transport_change_setup_target { my $guid = shift; my $dsf = shift; my $nodeid = shift; my $return_code = USM_SUCCESS; my @output; my @transports; my $rc; my $block_name; my $seqnum = 0; my $whoami = (caller(0))[3]; my $path = ""; my $iqn; my $hostname; my $transport_vip; my $single_transport = ""; lib_trace(9999, "Enter $whoami"); # The $nodeid is an optional argument for transport_change --export. If # it is specified, the export is only done via the specified transport. # Otherwise, all transports are used. if(defined ($nodeid)) { # If no specific transport was specified, create the target for all # transports. This variable would be left blank. $single_transport =~ /\w+\.(\w+\.\d+)/; push @transports, $single_transport; } else { # Expected output for 'advmutil transport list -g $guid' # clusterName: NSHGA2603 (clusterGUID: b9ba5d73a27bff74bf8db9073edc7d7a) # transportType sequenceNum # ----------------------------------------------------------- # iSCSI 0 # We are going to need to dump all transports for this guid. ($rc,@transports) = execute_command("$ADVMUTIL transport list " . " -g $guid | grep 'iSCSI.*[0-9]\\{1,\\}'" . "| xargs printf \"%s.%i\\\\n\""); # Expected output of this command after pipes # iSCSI.0 # iSCSI.1 } foreach $single_transport (@transports) { ($rc,@output) = execute_command("$ADVMUTIL transport list " . " -g $guid -v -i $single_transport"); for (@output) { my $line = $_; if($line =~ /\s*BLK|[iI]SCSI\s+(\d+)/) { $seqnum = $1; $iqn = iscsi_get_iqn($seqnum,$dsf); $block_name = iscsi_get_block_name($seqnum,$dsf); if ( -d "/sys/kernel/config/target/iscsi/$iqn" ) { lib_trace(9999, "Removing $iqn export"); osds_acfsr_transport_change_delete_target($guid,$dsf); } # Do not create the default 0.0.0.0 portal $return_code += run_targetcli($path, "set global " . "auto_add_default_portal=false"); $path = "/backstores/block/"; $return_code += run_targetcli($path, "create $block_name $dsf"); $path = "/iscsi/"; $return_code += run_targetcli($path, "create $iqn"); $path .= "$iqn/tpg1/"; $return_code += run_targetcli($path,"set attribute authentication=0"); $return_code += run_targetcli($path . "luns/", "create /backstores/block/$block_name"); } if($line =~ /\s*(\S+)\s*<\/BINDNAME>/ || $line =~ /\s*(\S+)\s*<\/BINDADDR>/) { $transport_vip = $1; # Use ping to determine fqdn my ($pingrc,@ping) = execute_command("ping $transport_vip -c 1| grep PING | cut -d' ' -f 2"); # Resolve IP address if($pingrc != 0) { lib_trace(9999, "unable to resolve transport ip address"); } my @addresses = gethostbyname($ping[0]); # filter out other data returned by gethostbyname @addresses = map { inet_ntoa($_) } @addresses[4 .. $#addresses]; my $transport_ip = shift @addresses; $return_code += run_targetcli($path . "portals/", "create $transport_ip 3260"); } } } return $return_code > 0 ? USM_FAIL : USM_SUCCESS; } # # This function deletes the iscsi target that matches the specified DSF for # the specified remote cluster. # Input: # - $guid = GUID of the cluster for which the storage is going to be # unexported. # - $dsf = Path to device special file being unexported. # Output: # Returns a counter of how many targetcli commands failed. 0 means # everything executed successfully. # sub osds_acfsr_transport_change_delete_target { my $guid = shift; my $dsf = shift; my $return_code = USM_SUCCESS; my $whoami = (caller(0))[3]; my $seqnum = 0; my $iqn; my $block_name; my $path = ""; my $attr; lib_trace(9999, "Enter $whoami"); my ($rc,@output) = execute_command("$ADVMUTIL transport list " . " -g $guid"); # Expected output for 'advmutil transport list -g $guid' # clusterName: NSHGA2603 (clusterGUID: b9ba5d73a27bff74bf8db9073edc7d7a) # transportType sequenceNum # ----------------------------------------------------------- # iSCSI 0 for (@output) { my $line = $_; if($line =~ /\s*BLK|[iI]SCSI\s+(\d+)/) { $seqnum = $1; $iqn = iscsi_get_iqn($seqnum,$dsf); $block_name = iscsi_get_block_name($seqnum,$dsf); $path = "/iscsi/"; $return_code += run_targetcli($path, "delete $iqn"); $path = "/backstores/block/"; $return_code += run_targetcli($path, "delete $block_name"); } } lib_trace(9999,"Exit $whoami with rc = $return_code"); return $return_code > 0 ? USM_FAIL : USM_SUCCESS; } sub run_targetcli { my $path = shift; my @args = @_; my $cmd = "/bin/targetcli"; my ($rc, @out) = execute_command("export HOME=/tmp; " . "$cmd set global " . "auto_save_on_exit=false 2>&1"); $cmd .= " $path"; for (@args) { $cmd .= " $_"; } ($rc,@out) = execute_command("export HOME=/tmp; " . $cmd); return $rc; } sub iscsi_get_block_name { my $seqnum = shift; my $dsf = shift; my $block_name = "acfsr"; my $mc = 0; my $mnr = 0; if($dsf =~ /.*\/(.*)-(.*)-(.*)/) { $mc = $1; $mnr = $3; } $block_name .= "-$mc-vol-$mnr"; lib_trace(9999,"Generated block name: $block_name"); return $block_name; } sub iscsi_get_iqn { my $seqnum = shift; my $dsf = shift; my $iqn = $iqn_pre; my $mc = 0; my $mnr = 0; if($dsf =~ /.*\/(.*)-(.*)-(.*)/) { $mc = $1; $mnr = $3; } $iqn .= "\\:$mc\\:$seqnum\\:vol-$mnr"; lib_trace(9999,"Generated IQN: $iqn"); return $iqn; } sub get_log_path { my $log_name = "acfsr_domain.log"; my ($ORACLE_BASE) = acfslib::getParam("ORACLE_BASE"); my $ORACLE_HOME = $acfslib::_ORACLE_HOME; my $hostname = qx/hostname -s/; my $log_dir; my $log_path; my $tmp_log_path; my $ret = 0; my $r = 0; my $w = 0; # getParam returns empty string if it fails get it from the environment $ORACLE_BASE = $ENV{ORACLE_BASE} if ($ORACLE_BASE eq ""); chomp ($hostname); ($hostname, undef) = split /\./, $hostname; # strip any domain name # Check ORACLE_BASE is set, if not check if crsdata is located in ORACLE_HOME if (!defined $ORACLE_BASE) { if (! -d "${ORACLE_HOME}/crsdata") { # We where unable to find the log path, fail lib_error_print(9509, "ORACLE_BASE is not set."); # Return a default log path, something like /usr/tmp/ should work as a # backup plan $log_dir = "/usr/tmp/"; } else { $log_dir = "${ORACLE_HOME}/crsdata/${hostname}/acfsrdiag"; } } else { $log_dir = "${ORACLE_BASE}/crsdata/${hostname}/acfsrdiag"; } # Check if the $log_directory exists, if not try to create it if (! -d $log_dir) { $ret = system ("mkdir -p $log_dir"); if ($ret) { lib_error_print(9345, "Unable to create directory: '%s'.",$log_dir); $log_dir = "/usr/tmp"; } } $log_path = "$log_dir/$log_name"; if (-e $log_path) { # Check current user permissions on logdir ($r,$w) = (-r $log_path, -w _); if (($r ne 1 || $w ne 1) && (! acfslib::lib_am_root())) { lib_error_print(9999, "Not enough permissions to access file: '%s'.", $log_path); $log_dir = "/usr/tmp"; lib_trace(9999, "Changing default log path to: '%s'.", $log_dir); } elsif (($r eq 1 || $w eq 1) && acfslib::lib_am_root ()) { # Change permissions on logdir $ret = system ("chmod 666 $log_path 2>/dev/null"); if ($ret) { lib_error_print(9999,"Unable to change permissions on file: '$log_path'."); $log_dir = "/usr/tmp"; lib_trace(9999, "Changing default log path to: '$log_dir'."); } } } else { # First call create acfsrlog file if (open ACFSRLOG, "> $log_path") { close ACFSRLOG; $ret = system ("chmod 666 $log_path 2>/dev/null"); if ($ret) { lib_error_print(9999,"Unable to change permissions on file: '$log_path'."); $log_dir = "/usr/tmp"; lib_trace(9999, "Changing default log path to: '$log_dir'."); } } else { lib_error_print(9295, "failed to open file %s", $log_path); } } if ($log_dir !~ /^\/usr\/tmp/) { # TEMPORARY log path, we want to avoid writing to this log path # in case the ORACLE_BASE is not set and we write all the time to this # log file $tmp_log_path = "/usr/tmp/"; # Check if the $log_directory exists, if not try to create it if (! -d $tmp_log_path) { $ret = system ("mkdir -p $tmp_log_path"); if ($ret) { lib_error_print(9345,"Unable to create directory: '%s'.",$tmp_log_path); } } if (-d $tmp_log_path) { $tmp_log_path .= "/$log_name"; if (open TMP, "> $tmp_log_path") { print TMP "acfsremote logs now in $log_path\n"; close TMP; } else { lib_trace(9295, "failed to open file %s", $tmp_log_path); } } } return $log_path; } 1;