Getting a variable from a field using arcpy - python-2.7

I am creating a toolbox tool using a python script that creates maps based on user input. I have created a map template that gets saved and altered with python. I am struggling on how to update some text in text boxes in the layout view using Arcpy. I was able to do it with dynamic text with data driven pages, but I couldn't find any python code to get data driven pages to refresh so I decided to try to update the text with python directly. With data driven pages, the dynamic text was pulling the text from an attribute table. I'm fairly new to python so am struggling with how to pull values from a table to use as part of the text. I am able to update text as long as I have the variable defined somewhere else (not from a table), but the only method I found to pull data from a table was with a search cursor but that returns a list rather than a value so I get an error. The feature classes with the text values I want only have one row in them so it is a list of one. How can I convert that list to a value. I am only including the applicable parts of the script. I also removed the actual paths from the code.
import arcpy
import os
ID = arcpy.GetParameterAsText(1)
city = arcpy.GetParameterAsText(3)
WS = os.path.join('path to gdb', "WS")
dfield = 'name'
datefield = 'date'
cfield = "county"
#Use SearchCursor - these features only have one row, but these are my problem because they are lists
wsname = [row[0] for row in arcpy.da.SearchCursor(WS, dfield)]
wsdate = [row[0] for row in arcpy.da.SearchCursor(WS, datefield)]
county = [row[0] for row in arcpy.da.SearchCursor(overview, cfield)]
#update text
for elm in arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT"):
elm.text = elm.text.replace('WS',wsname) #this doesn't work because wsname is a list
elm.text = elm.text.replace('City',city) #this works
elm.text = elm.text.replace('text2',"words"+ ID +" -more words") #This works
elm.text = elm.text.replace('Name', county) #this doesn't work because county is a list
elm.text = elm.text.replace('Date',wsdate) #this doesn't work because wsdate is a list
arcpy.RefreshActiveView()
mxd.save()

This code will work when run from a arcgis toobox script tool.
# define the aprx file and the layout in the project
import arcpy
aprx = arcpy.mp.ArcGISProject(r'path\to\the\arcgis\aprxfile.aprx')
aprxLayout = aprx.listLayouts()[0] '''adding the zero index will return the first
layout in the layout list, if there is more than one layout'''
# get the attribute value to use for the layout text element
fieldNames = ['FieldName1', 'FieldName2']
with arcpy.da.SearchCursor(r'path\to.gdb\featureclass', fieldNames) as sc:
for row in sc:
if (row[0]) is not None:
field1Value = (row[0])
if (row[0]) is None:
field1Value = 'Null'
if (row[1]) is not None:
field2Value = (row[0])
if (row[1]) is None:
field2Value = 'Null'
# Assign the attribute value to the layout text element
for textElem in aprxLayout.listElements:
if textElem.name == 'name of layout text element in the element properties':
text.Elem.text = field1Value
if textElem.name == 'name of layout text element in the element properties':
text.Elem.text = field2Value
aprx.saveACopy(r'path/to/folder/projectname')
del aprx

I was able to tweak armedwiththeword's code to come up with this.
import arcpy
mxd = arcpy.mapping.MapDocument(path_to_mxd)
fieldNames = ['name', 'date']
with arcpy.da.SearchCursor(WS, fieldNames) as sc:
for row in sc:
if(row[0]) is not None:
field1Value = (row[0])
if(row[0]) is None:
field1Value = 'Null'
if(row[1]) is not None:
field2Value = (row[1])
if(row[1]) is None:
field2Value = 'Null'
fieldName = ['CTY_NAME']
with arcpy.da.SearchCursor(overview, fieldName) as sc:
for row in sc:
if(row[0]) is not None:
field3Value = (row[0])
if(row[0]) is None:
field3Value = 'Null'
# Assign the attribute value to the layout text element
for textElem in arcpy.mapping.ListLayoutElements(mxd,'TEXT_ELEMENT'):
if textElem.name == 'title':
textElem.text = field1Value + " words"
if textElem.name == 'subtitle':
textElem.text = "WS -0"+ ID + " -more words"
if textElem.name == 'city':
textElem.text = city
if textElem.name == 'county':
textElem.text = field3Value
if textElem.name == 'date':
textElem.text = field2Value

Related

Why does the render_template keep on showing the old value of the flask form?

I've been searching for an answer for hours. I apologise if I missed something.
I'm using the same form multiple times in order to add rows to my database.
Every time I check an excel file to pre-fill some of the wtforms StringFields with known information that the user may want to change.
The thing is: I change the form.whatever.data and when printing it, it shows the new value. But when I render the template it keeps showing the old value.
I tried to do form.hours_estimate.data = "" before assigning it a new value just in case but it didn't work.
I will attach here the route I'm talking about. The important bit is after # Get form ready for next service. If there's more info needed please let me know.
Thank you very much.
#coordinator_bp.route("/coordinator/generate-order/<string:pev>", methods=['GET', 'POST'])
#login_required
def generate_order_services(pev):
if not (current_user.is_coordinator or current_user.is_manager):
return redirect(url_for('public.home'))
# Get the excel URL
f = open("./app/database/datafile", 'r')
filepath = f.read()
f.close()
error = None
if GenerateServicesForm().submit1.data and GenerateServicesForm().validate():
# First screen submit (validate the data -> first Service introduction)
form = FillServiceForm()
next_service_row = get_next_service_row(filepath)
if next_service_row is None:
excel_info = excel_get_pev(filepath)
error = "Excel error. Service code not found. If you get this error please report the exact way you did it."
return render_template('coordinator/get_pev_form.html', form=GetPevForm(), error=error, info=excel_info)
service_info = get_service_info(filepath, next_service_row)
service_code = service_info[0]
start_date = service_info[1]
time_estimate = service_info[2]
objects = AssemblyType.get_all()
assembly_types = []
for assembly_type in objects:
assembly_types.append(assembly_type.type)
form.service_code.data = service_code
form.start_date.data = start_date
form.hours_estimate.data = time_estimate
return render_template('coordinator/fill_service_form.html', form=form, error=error, assembly_types=assembly_types)
if FillServiceForm().submit2.data:
if not FillServiceForm().validate():
objects = AssemblyType.get_all()
assembly_types = []
for assembly_type in objects:
assembly_types.append(assembly_type.type)
return render_template('coordinator/fill_service_form.html', form=FillServiceForm(), error=error,
assembly_types=assembly_types)
# Service screen submits
# Here we save the data of the last submit and ready the next one or end the generation process
# Ready the form
form = FillServiceForm()
next_service_row = get_next_service_row(filepath)
if next_service_row is None:
excel_info = excel_get_pev(filepath)
error = "Excel error. Service code not found. If you get this error please report the exact way you did it."
return render_template('coordinator/get_pev_form.html', form=GetPevForm(), error=error, info=excel_info)
service_info = get_service_info(filepath, next_service_row)
service_code = service_info[0]
form.service_code.data = service_code
# create the service (this deletes the service code from the excel)
service = create_service(form, filepath)
if isinstance(service,str):
return render_template('coordinator/fill_service_form.html', form=form, error=service)
# Get next service
next_service_row = get_next_service_row(filepath)
if next_service_row is None:
# This means there is no more services pending
return "ALL DONE"
else:
# Get form ready for next service
service_info = get_service_info(filepath, next_service_row)
service_code = service_info[0]
start_date = service_info[1]
time_estimate = service_info[2]
print("time_estimate")
print(time_estimate) # I get the new value.
objects = AssemblyType.get_all()
assembly_types = []
for assembly_type in objects:
assembly_types.append(assembly_type.type)
form.service_code.data = service_code
form.start_date.data = start_date
form.hours_estimate.data = time_estimate
print(form.hours_estimate.data) # Here I get the new value. Everything should be fine.
# In the html, the old value keeps on popping.
return render_template('coordinator/fill_service_form.html', form=form, error=error,
assembly_types=assembly_types)
number_of_services = excel_get_services(filepath=filepath, selected_pev=pev)
# Get the number of the first excel row of the selected pev
first_row = excel_get_row(filepath, pev)
if first_row is None:
excel_info = excel_get_pev(filepath)
error = "Excel error. PEV not found. If you get this error please report the exact way you did it."
return render_template('coordinator/get_pev_form.html', form=GetPevForm(), error=error, info=excel_info)
service_code = []
start_date = []
time_estimate_code = []
quantity = []
# Open the excel
wb = load_workbook(filepath)
# grab the active worksheet
ws = wb.active
for idx in range(number_of_services):
# Append the data to the lists
service_code.append(ws.cell(row=first_row+idx, column=12).value)
start_date.append(str(ws.cell(row=first_row + idx, column=5).value)[:10])
time_estimate_code.append(ws.cell(row=first_row+idx, column=7).value)
quantity.append(ws.cell(row=first_row + idx, column=9).value)
wb.close()
return render_template('coordinator/generate_services_form.html',
form=GenerateServicesForm(),
pev=pev,
service_code=service_code,
start_date=start_date,
time_estimate_code=time_estimate_code,
quantity=quantity)
Well I found a workarround: I send the data outside the form like this:
return render_template('coordinator/fill_service_form.html', form=form, error=error,
assembly_types=assembly_types,
service_code=service_code,
start_date=start_date,
time_estimate=time_estimate)
And replace the jinja form for this:
<input class="form-control" placeholder="2021-04-23" name="start_date" type="text" value="{{start_date}}">
I'm still using the form (name= the form field name) and at the same time I input the value externally.
I hope this helps somebody.

IMPORTING CSV DATAS INTO MODEL - DJANGO

I'm implementing a list of clients. I want to give to the user the possibility of importing new clients thought a csv file.
The client model has these fields: Client, name, surname, email, phone
So I created this model:
class CsvClient(models.Model):
file_name = models.FileField(upload_to='csv-cliente')
uploaded = models.DateTimeField(auto_now_add=True)
activated = models.BooleanField(default=False)
def __str__(self):
return f"File id: {self.id}"
and this function in views.py:
import csv
def importa_csv_clienti(request):
form = CVSForm(request.POST or None, request.FILES or None)
if form.is_valid():
form.save()
form = CVSForm()
clients = CsvClient.objects.get(activated=False)
with open(clienti.file_name.path, 'r') as f:
reader = csv.reader(f)
for i, row in enumerate(reader):
if i==0:
pass
else:
row = "".join(row)
row = row.replace(";", " ")
row = row.split(" ")
client = row[0].capitalize()
name = row[1].capitalize()
surname = row[2].capitalize()
value = Cliente.objects.create(
cliente=cliente,
nome=nome,
cognome=cognome,
email=riga[3],
telefono=riga[4],
)
print('oggetto creato:', value.cliente, value.nome, value.cognome, value.email, value.telefono)
clients.activated = True
clients.save()
context = {'form': form}
template = 'importa.html'
return render(request, template, context)
It works, expect for the fact that if in the csv file I have the row:
Nutella Antonio Dello Iudice
where
nutella is client
Antonio is name
Dello Iodice is surname
and email and phone are blank
basically it interprets it as if
Dello is the surname and Iudice is the email.
How do I tell him not to separate the surname if it composed by 2 or more words?
PS: this a curiosity, but how do I Know if the csv file imported by the user is separated by ; and not , ? Because I set as separator ";", but if the user imports a file that is separated by "," the my code won't work, right?
Right now you are replacing ";" by a whitespace. So a row like Nutella;Antonio Dello; Iudice would be converted to Nutella Antonio Dello Iudice.
After this replacement you do a split for whitespace.
Why not removing row = row.replace(";", " ")and just doing row = row.split(";") ?
If I understood your question right, you want to check what separator is used right?
One simple method would be to check the file for ; - and if the count of them is bigger or equals 4 (0;1;2;3;4) the separator is ;.
For the first question about splitting you surname:
row = "".join(row)
row = row.replace(";", " ")
row = row.split(" ")
Why do you replace ; with blank spaces? Just split the row by you separator, since in a Valid CSV it is not allowed to use the separator inside the content.
row = "".join(row)
row = row.split(";")
Now your surname is not split when containing a blankspace.

Django : Aggregate Sum works on view only after a refresh

I have an Invoice model in Django that has multiple Line models.(line items that have a title, unit price , and qty)
I have a 'addline' view that allows to add item lines to the invoice.
The view also displays all current invoice lines and a calculated sum of the total price for all line items.
when I submit a new line item, the view refreshes to the same page and the line item appears properly, but the total (totalservices or totalgoods) of line items is not updated .
It becomes updated when i refresh the page manually , or when i add another line -with the previous line total.
here is my relevant view
def addline(request, id):
form = AddLineForm(request.POST or None )
invoice = get_object_or_404(Invoice, id = id)
linelist = Line.objects.filter(invoice = id).order_by('created_at')
servicelines = linelist.filter(line_type = "S")
goodslines = linelist.filter(line_type = "G")
totalservice = servicelines.aggregate(Sum('line_total'))['line_total__sum']
print servicelines.aggregate(Sum('line_total'))
totalgoods = goodslines.aggregate(Sum('line_total'))['line_total__sum']
if form.is_valid():
instance = form.save(commit=False) #do schtuff with data
instance.invoice = invoice
instance.line_total = instance.unit_price *instance.qty
#print instance.line_total
if form.cleaned_data.get('overwrite'):
invoice.invoiced_service = totalservice or 0 #or zero to prevent fuss if list is empty
invoice.invoiced_goods = totalgoods or 0
invoice.save()
form.save()
form = AddLineForm()
context = {'inv': invoice, 'form': form, 'lines':linelist, 'goods': goodslines, 'services': servicelines ,'totalservice' : totalservice, 'totalgoods':totalgoods }
return render(request,'testpaper.html', context)
Thanks in advance, I'm not sure what could be the problem. Maybe that the Sum is lazy and not evaluated ?
*edited to reflect actual view
Found the problem : the 'if form.is_valid 'block that ends in form.save() needed to be before the block that calculates lines , otherwise lines are calculated before current item has been saved .

How to access the values inside the 'files' field in scrapy

I have downloaded some files using the file pipeline and i want to get the values of the files field. I tried to print item['files'] and it gives me a key error. Why is this so and how can i do it?
class testspider2(CrawlSpider):
name = 'genspider'
URL = 'flu-card.com'
URLhttp = 'http://www.flu-card.com'
allowed_domains = [URL]
start_urls = [URLhttp]
rules = (
[Rule(LxmlLinkExtractor(allow = (),restrict_xpaths = ('//a'),unique = True,),callback='parse_page',follow=True),]
)
def parse_page(self, response):
List = response.xpath('//a/#href').extract()
item = GenericspiderItem()
date = strftime("%Y-%m-%d %H:%M:%S")#get date&time dd-mm-yyyy hh:mm:ss
MD5hash = '' #store as part of the item, some links crawled are not file links so they do not have values on these fields
fileSize = ''
newFilePath = ''
File = open('c:/users/kevin123/desktop//ext.txt','a')
for links in List:
if re.search('http://www.flu-card.com', links) is None:
responseurl = re.sub('\/$','',response.url)
url = urljoin(responseurl,links)
else:
url = links
#File.write(url+'\n')
filename = url.split('/')[-1]
fileExt = ''.join(re.findall('.{3}$',filename))
if (fileExt != ''):
blackList = ['tml','pdf','com','php','aspx','xml','doc']
for word in blackList:
if any(x in fileExt for x in blackList):
pass #url is blacklisted
else:
item['filename'] = filename
item['URL'] = url
item['date'] = date
print item['files']
File.write(fileExt+'\n')
yield GenericspiderItem(
file_urls=[url]
)
yield item
It is not possible to access item['files'] in your spider. That is because the files are download by the FilesPipeline, and items just reach pipelines after they get out of your spider.
You first yield the item, then it gets to FilesPipeline, then the files are dowloaded, an just then the field images is populated with the info you want. To access it, you have to write a pipeline and schedule it after the FilesPipeline. Inside your pipeline, you can access the files field.
Also note that, in your spider, you are yielding to different kinds of items!

GeoDJango: retrieve last inserted primary key from LayerMapping

I am building an application with GeoDjango and I have the following problem:
I need to read track data from a GPX file and those data should be stored in a model MultiLineStringField field.
This should happen in the admin interface, where the user uploads a GPX file
I am trying to achieve this, namely that the data grabbed from the file should be assigned to the MultiLineStringField, while the other fields should get values from the form.
My model is:
class GPXTrack(models.Model):
nome = models.CharField("Nome", blank = False, max_length = 255)
slug = models.SlugField("Slug", blank = True)
# sport natura arte/cultura
tipo = models.CharField("Tipologia", blank = False, max_length = 2, choices=TIPOLOGIA_CHOICES)
descrizione = models.TextField("Descrizione", blank = True)
gpx_file = models.FileField(upload_to = 'uploads/gpx/')
track = models.MultiLineStringField(blank = True)
objects = models.GeoManager()
published = models.BooleanField("Pubblicato")
rel_files = generic.GenericRelation(MyFiles)
#publish_on = models.DateTimeField("Pubblicare il", auto_now_add = True)
created = models.DateTimeField("Created", auto_now_add = True)
updated = models.DateTimeField("Updated", auto_now = True)
class Meta:
#verbose_name = "struttura'"
#verbose_name_plural = "strutture"
ordering = ['-created']
def __str__(self):
return str(self.nome)
def __unicode__(self):
return '%s' % (self.nome)
def put(self):
self.slug = sluggy(self.nome)
key = super(Foresta, self).put()
# do something after save
return key
While in the admin.py file I have overwritten the save method as follows:
from django.contrib.gis import admin
from trails.models import GPXPoint, GPXTrack
from django.contrib.contenttypes import generic
from django.contrib.gis.gdal import DataSource
#from gpx_mapping import GPXMapping
from django.contrib.gis.utils import LayerMapping
from django.template import RequestContext
import tempfile
import os
import pprint
class GPXTrackAdmin(admin.OSMGeoAdmin):
list_filter = ( 'tipo', 'published')
search_fields = ['nome']
list_display = ('nome', 'tipo', 'published', 'gpx_file')
inlines = [TrackImagesInline, TrackFilesInline]
prepopulated_fields = {"slug": ("nome",)}
def save_model(self, request, obj, form, change):
"""When creating a new object, set the creator field.
"""
if 'gpx_file' in request.FILES:
# Get
gpxFile = request.FILES['gpx_file']
# Save
targetPath = tempfile.mkstemp()[1]
destination = open(targetPath, 'wt')
for chunk in gpxFile.chunks():
destination.write(chunk)
destination.close()
#define fields of interest for LayerMapping
track_point_mapping = {'timestamp' : 'time',
'point' : 'POINT',
}
track_mapping = {'track' : 'MULTILINESTRING'}
gpx_file = DataSource(targetPath)
mytrack = LayerMapping(GPXTrack, gpx_file, track_mapping, layer='tracks')
mytrack.save()
#remove the temp file saved
os.remove(targetPath)
orig = GPXTrack.objects.get(pk=mytrack.pk)
#assign the parsed values from LayerMapping to the appropriate Field
obj.track = orig.track
obj.save()
As far as I know:
LayerMapping cannot be used to update a field but only to save a new one
I cannot access a specific field of the LayerMapping object (ie in the code above: mytrack.track) and assign its value to a model field (ie obj.track) in the model_save method
I cannot retrieve the primary key of the last saved LayerMapping object (ie in the code above: mytrack.pk) in order to update it with the values passed in the form for the field not mapped in LayerMapping.mapping
What can I do then?!?!
I sorted it out subclassing LayerMapping and adding a method get_values() that instead of saving the retrieved data, returns them for any use or manipulation.The get_values method is a copy of the LayerMapping::save() method that returns the values instead of saving them.
I am using django 1.5
import os
from django.contrib.gis.utils import LayerMapping
import sys
class MyMapping(LayerMapping):
def get_values(self, verbose=False, fid_range=False, step=False,
progress=False, silent=False, stream=sys.stdout, strict=False):
"""
Returns the contents from the OGR DataSource Layer
according to the mapping dictionary given at initialization.
Keyword Parameters:
verbose:
If set, information will be printed subsequent to each model save
executed on the database.
fid_range:
May be set with a slice or tuple of (begin, end) feature ID's to map
from the data source. In other words, this keyword enables the user
to selectively import a subset range of features in the geographic
data source.
step:
If set with an integer, transactions will occur at every step
interval. For example, if step=1000, a commit would occur after
the 1,000th feature, the 2,000th feature etc.
progress:
When this keyword is set, status information will be printed giving
the number of features processed and sucessfully saved. By default,
progress information will pe printed every 1000 features processed,
however, this default may be overridden by setting this keyword with an
integer for the desired interval.
stream:
Status information will be written to this file handle. Defaults to
using `sys.stdout`, but any object with a `write` method is supported.
silent:
By default, non-fatal error notifications are printed to stdout, but
this keyword may be set to disable these notifications.
strict:
Execution of the model mapping will cease upon the first error
encountered. The default behavior is to attempt to continue.
"""
# Getting the default Feature ID range.
default_range = self.check_fid_range(fid_range)
# Setting the progress interval, if requested.
if progress:
if progress is True or not isinstance(progress, int):
progress_interval = 1000
else:
progress_interval = progress
# Defining the 'real' save method, utilizing the transaction
# decorator created during initialization.
#self.transaction_decorator
def _get_values(feat_range=default_range, num_feat=0, num_saved=0):
if feat_range:
layer_iter = self.layer[feat_range]
else:
layer_iter = self.layer
for feat in layer_iter:
num_feat += 1
# Getting the keyword arguments
try:
kwargs = self.feature_kwargs(feat)
except LayerMapError, msg:
# Something borked the validation
if strict: raise
elif not silent:
stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg))
else:
# Constructing the model using the keyword args
is_update = False
if self.unique:
# If we want unique models on a particular field, handle the
# geometry appropriately.
try:
# Getting the keyword arguments and retrieving
# the unique model.
u_kwargs = self.unique_kwargs(kwargs)
m = self.model.objects.using(self.using).get(**u_kwargs)
is_update = True
# Getting the geometry (in OGR form), creating
# one from the kwargs WKT, adding in additional
# geometries, and update the attribute with the
# just-updated geometry WKT.
geom = getattr(m, self.geom_field).ogr
new = OGRGeometry(kwargs[self.geom_field])
for g in new: geom.add(g)
setattr(m, self.geom_field, geom.wkt)
except ObjectDoesNotExist:
# No unique model exists yet, create.
m = self.model(**kwargs)
else:
m = self.model(**kwargs)
try:
# Attempting to save.
pippo = kwargs
num_saved += 1
if verbose: stream.write('%s: %s\n' % (is_update and 'Updated' or 'Saved', m))
except SystemExit:
raise
except Exception, msg:
if self.transaction_mode == 'autocommit':
# Rolling back the transaction so that other model saves
# will work.
transaction.rollback_unless_managed()
if strict:
# Bailing out if the `strict` keyword is set.
if not silent:
stream.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid)
stream.write('%s\n' % kwargs)
raise
elif not silent:
stream.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg))
# Printing progress information, if requested.
if progress and num_feat % progress_interval == 0:
stream.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved))
# Only used for status output purposes -- incremental saving uses the
# values returned here.
return pippo
nfeat = self.layer.num_feat
if step and isinstance(step, int) and step < nfeat:
# Incremental saving is requested at the given interval (step)
if default_range:
raise LayerMapError('The `step` keyword may not be used in conjunction with the `fid_range` keyword.')
beg, num_feat, num_saved = (0, 0, 0)
indices = range(step, nfeat, step)
n_i = len(indices)
for i, end in enumerate(indices):
# Constructing the slice to use for this step; the last slice is
# special (e.g, [100:] instead of [90:100]).
if i + 1 == n_i: step_slice = slice(beg, None)
else: step_slice = slice(beg, end)
try:
pippo = _get_values(step_slice, num_feat, num_saved)
beg = end
except:
stream.write('%s\nFailed to save slice: %s\n' % ('=-' * 20, step_slice))
raise
else:
# Otherwise, just calling the previously defined _save() function.
return _get_values()
In a custom save or save_model method you can then use:
track_mapping = {'nome': 'name',
'track' : 'MULTILINESTRING'}
targetPath = "/my/gpx/file/path.gpx"
gpx_file = DataSource(targetPath)
mytrack = MyMapping(GPXTrack, gpx_file, track_mapping, layer='tracks')
pippo = mytrack.get_values()
obj.track = pippo['track']