importfromdatasource Job File
The opcli importfromdatasource command uses a YAML job file to control its behavior.
The job file allows control of the following, among other things:
What data source to use: a JSON file, a ServiceNow table, or a URL
How to search/filter records pulled from the source and the OpsRamp resource records they are enriching
How to match source record with OpsRamp resource records (multiple strategies can be used)
What OpsRamp native and custom resource attributes are enriched
Renames that must occur between the source attributes and the target OpsRamp attributes
Value mappings or value transformations that need to occur
How to determine duplicate resource records in OpsRamp, and whether or not to delete them
How to identify duplicate source records, and how to determine the correct one to use
Whether or not to delete OpsRamp resources with no matches in the source data
Whether or not to create missing OpsRamp resources when they are present in the source
Sample job file
# Define the external data source. Current supported sources are servicenow, urlsource, and jsonsource
# You can only use one type, the others must be removed or commented out.
# JSON file source
# The source file is expected to contain an array of objects.
source:
jsonsource:
filename: "/my/path/to/mysourcefile.json"
result_key: result
# ServiceNow data source
# source:
# servicenow:
# instance_url: https://myinstance.service-now.com
# auth:
# type: basic
# username: myuser
# password: mypassword
# table: cmdb_ci_linux_server
# query_parameters:
# query: put_a_servicenow_encoded_query_here
# fields: name,os,os_version,os_service_pack,ip_address,operational_status,sys_updated_on
# display_value: true
# limit: 100000
# URL data source
# source:
# urlsource:
# url: https://myurl.mydomain.com
# result_key: result
# auth:
# type: basic
# username: myuser
# password: mypassword
# query_parameters:
# sysparm_query: nameLIKElnux
# sysparm_fields: name,os,os_version,os_service_pack,ip_address,operational_status,sys_updated_on
# sysparm_display_value: true
# sysparm_limit: 100000
######################################
# Define the OpsRamp opsql query to retrieve OpsRamp resources that will be matched/reconciled with
# the records coming from the external data source
resource_opsql:
pageSize: 1000
fields:
- id
- name
- tags
- ipAddress
- hostName
- resourceName
- type
- nativeType
- aliasName
filter: ""
sortBy: created
# Define one or more match strategies to be used for matching external records with the retrieved OpsRamp resources
#
# If you want duplicate resource in OpsRamp matching the same criteria to be deleted (other than the first found match)
# then specify "duplicate_handling: delete_duplicates" for the strategy.
#
# If you want to update all of the dups with the same attributes as the original, then specify "duplicate_handling: update_all"
#
# Otherwise you can specify "duplicate_handling: ignore" and the dups will be left alone
match_strategy:
- name: Exact name after stripping fqdn
opsramp_fields:
- fieldname: name
normalize:
regex: "\\..*\\.(?:com|net)$"
replace: ""
source_fields:
- fieldname: resourceName
normalize:
regex: "\\..*\\.(?:com|net)$"
replace: ""
duplicate_handling: update_all
- name: Exact resourceName after stripping fqdn
opsramp_fields:
- fieldname: resourceName
normalize:
regex: "\\..*\\.(?:com|net)$"
replace: ""
source_fields:
- fieldname: resourceName
normalize:
regex: "\\..*\\.(?:com|net)$"
replace: ""
duplicate_handling: update_all
- name: Exact hostName after stripping fqdn
opsramp_fields:
- fieldname: hostName
normalize:
regex: "\\..*\\.(?:com|net)$"
replace: ""
source_fields:
- fieldname: resourceName
normalize:
regex: "\\..*\\.(?:com|net)$"
replace: ""
duplicate_handling: update_all
- name: Match by IP address
opsramp_fields:
- fieldname: ipAddress
source_fields:
- fieldname: ip_address
duplicate_handling: update_all
# Define static field values to be added for all external records
static_fieldvalue:
client.uniqueId: 4e6ab0d9-dd8d-489b-97f5-bf92659e65d9
client.name: Prod
tags.last_urlsource_sync: DATEUTC()
tags.operational_status: Operational
# Define mappings of source field names to OpsRamp resource field names in the form source_name: opsramp_name
# Note that tags.fieldname denotes an OpsRamp custom attribute with a name of fieldname.
fieldname_mapping:
name: resourceName
# aliases: tags.sn_aliases
app_criticality: tags.sn_app_criticalities
applications: tags.sn_applications
assigned_to: tags.sn_assigned_to
assigned_to_vip: tags.sn_assigned_to_vip
assigned_to_vip_status: tags.sn_assigned_to_vip_status
carrier: tags.sn_carrier
chassis: tags.sn_chassis
chef_roles: tags.sn_chef_roles
circuits: tags.sn_circuits
datacenter_loc: tags.sn_datacenter_loc
db_instances: tags.sn_db_instances
device_type: tags.sn_device_type
environment: tags.sn_env
escalation_exceptions: tags.sn_escalation_exceptions
escalation_exceptions_desc: tags.sn_escalation_exceptions_desc
esx_host: tags.sn_esx_server
# ip_address: tags.sn_ip_address
# ip_connections: tags.sn_ip_connections
location_city: tags.sn_loc_city
location_country: tags.sn_loc_country
location_region: tags.sn_loc_region
location_site_id: tags.sn_loc_site_id
location_state: tags.sn_loc_state
location_street: tags.sn_loc_street
location_time_zone: tags.sn_loc_timezone
location_zip: tags.sn_loc_zip
model_id: tags.sn_model_id
notification_contacts: tags.sn_notification_contacts
ops_genie_group_id: tags.sn_ops_genie_group_id
ops_genie_group_name: tags.sn_ops_genie_group_name
os_model: tags.sn_os
os_service_pack: tags.sn_os_service_pack
owned_by: tags.sn_owned_by
primary_application: tags.sn_primary_application
primary_application_criticality: tags.sn_primary_application_criticality
production_level: tags.sn_production_level
rack: tags.sn_rack
reboot_custom: tags.sn_reboot_custom
reboot_day_of_week: tags.sn_reboot_day_of_week
reboot_hour: tags.sn_reboot_hour
reboot_minute: tags.sn_reboot_minute
reboot_month_of_quarter: tags.sn_reboot_month_of_quarter
reboot_period: tags.sn_reboot_period
reboot_week_of_month: tags.sn_reboot_week_of_month
# serial_number: tags.sn_serial_number
support_group: tags.sn_support_group
sys_class_name: resourceType
# Sometimes you need to use a source field in more than one place. If thats the case
# you can specify that here as newfield: sourcefield and the value will bew copied to newfield.
# Make sure to use the renamed field names if you have specified any of them in the
# fieldname_mapping section. The value will be copied before any fieldvalue_mapping
# translations are done.
copy_fields:
tags.sn_sys_class_name: resourceType
aliasName: resourceName
# If you wish to CREATE resources for any source records where an existing matching resource
# is not found in the specified opsql query, specify here the native attribute fields you wish
# to use in creating them. These specified native fields plus all of the defined custom
# attributes (i.e. tags.* fields) will be populsted.
# If you don't want to create missing resources, just comment out the section.
create_missing_resources:
native_attrs:
- resourceName
- resourceType
# If you wish to UPDATE any resource native attributes for source records where a match is
# found in the specified opsql query, specify here the native attribute fields you wish
# to uget updated. Only these specified native fields plus all of the defined custom
# attributes (i.e. tags.* fields) will be updated.
# If you don't want to update native attributes on matched resources, just comment out the section.
#
# update_existing_resources:
# native_attrs:
# - resourceName
# - resourceType
# If you wish to DELETE existing OpsRamp resources found via the that do not exist in source data set
# uncomment this section. You can specify one or more OpsRamp resourceTypes to be excepted from this
# as well as one or more resourceNames to be excepted. Note that the exceptions defined here will not
# affect duplicate deletion specified in the match_strategy section.
# Most commonly, delete exceptions are needed for items found/created via OpsRamp's native discovery but
# not tracked in the external data source (such as CMDB).
delete_existing_resources_if_not_in_source:
except_for_matching:
type:
- Gateway
- DATA_CENTER
- VI_DATASTORE
- VI_CLUSTER
- CISCO_MERAKI_NETWORK
- VOIP Device
- CLOUD_PROVIDER
- iDRAC
- Management Controller
- Load Balancer
- SYNTHETIC
- Cluster
- WLAN Controller
- MERAKI_ORGANIZATION
- DOCKER_CONTAINER
- BSD
nativeType:
- OpsRamp Gateway
- HLE ILO
- Nutanix Disk
- Nutanix Storage Container
- Nutanix Controller VM
- Nutanix Cluster
- Nutanix Cluster Manager
- Aruba Mobility Controller
- VMware Datacenter
- VMware Datastore
- VMware Cluster
- Networks
- HTTPS
- PING
- Organizations
- Other Devices
- Docker Container
except_for_resources: []
# By default, custom attribute (i.e. tags.* field) values populated will not be set as metric
# labels. If you want them to be set as metric labels, specify the custom attribute names here,
# omitting the "tags." portion of the name and in the format custattrfieldname: true
tag_as_metric_label: []
# If you need to translate or map the values of ant vields, you can do so here. Provide a list of
# regex expressions and replace strings and they will be applied to the specified field.
fieldvalue_mapping:
aliasName:
- regex: "\\..*\\.(?:com|net)$"
replace: ""
tags.region:
- regex: (?i)^us.*
replace: americas
- regex: (?i)^ap.*
replace: apac
- regex: (?i)^eu.*
replace: europe
resourceType:
- regex: cmdb_ci_chassis_server
replace: Management Controller
- regex: cmdb_ci_comm
replace: Other
- regex: cmdb_ci_computer
replace: Work Station
- regex: cmdb_ci_esx_server
replace: VMWAREHOST
- regex: cmdb_ci_firewall_network
replace: Firewall
- regex: cmdb_ci_hardware
replace: Other
- regex: cmdb_ci_linux_server
replace: Linux
- regex: cmdb_ci_msd
replace: Other
- regex: cmdb_ci_net_app_server
replace: Storage
- regex: cmdb_ci_netgear
replace: Network Device
- regex: cmdb_ci_printer
replace: Other
- regex: cmdb_ci_storage_controller
replace: Storage
- regex: cmdb_ci_storage_device
replace: Storage
- regex: cmdb_ci_ups
replace: Other
- regex: cmdb_ci_voice_hardware
replace: VOIP Device
- regex: cmdb_ci_win_server
replace: Windows
- regex: u_appliance
replace: Other
- regex: u_network_circuits
replace: Other
- regex: u_security_device
replace: Network Device
- regex: u_tape_library
replace: Storage
- regex: cmdb_ci_vm
replace: Other
# If there are any old or unused custom attributes that you want to be removed from all
# resources, provide a list of them here.
clear_attribute:
- sys_class_name
# If you want to ignore or filter out and records from the source data, specify a source field name
# and a list of values where those records should be ignored.
exceptions:
resourceName: []