why my function doesn't remove objects from final_list? - python-2.7

Question

hey guys:)im sorry for all the code but i feel its necessary for u to see everything..
i tried everything... i hidden prints in the code, debugging for ten times, triple checked the built in methods, and still, the .crawl() method dosnt remove any object from the final_list.
the object of my assignment is to built two classes:
Web_page : holds data of a web page.(the pages come in the form of html files saved in a folder on my desktop. Crawler: compare between pages and hold a list of the uniqe pages---> final_list
import re
import os
def remove_html_tags(s):
tag = False
quote = False
out = ""
for c in s:
if c == '<' and not quote:
tag = True
elif c == '>' and not quote:
tag = False
elif (c == '"' or c == "'") and tag:
quote = not quote
elif not tag:
out = out + c
return out
def lev(s1, s2):
return lev_iter(s1, s2, dict())
def lev_iter(s1, s2, mem):
(i,j) = (len(s1), len(s2))
if (i,j) in mem:
return mem[(i,j)]
s1_low = s1.lower()
s2_low = s2.lower()
if len(s1_low) == 0 or len(s2_low) == 0:
return max(len(s1_low), len(s2_low))
d1 = lev_iter(s1_low[:-1], s2_low, mem) + 1
d2 = lev_iter(s1_low, s2_low[:-1], mem) + 1
last = 0 if s1_low[-1] == s2_low[-1] else 1
d3 = lev_iter(s1_low[:-1], s2_low[:-1], mem) + last
result = min(d1, d2, d3)
mem[(i,j)] = result
return result
def merge_spaces(content):
return re.sub('\s+', ' ', content).strip()
""" A Class that holds data on a Web page """
class WebPage:
def __init__(self, filename):
self.filename = filename
def process(self):
f = open(self.filename,'r')
LINE_lst = f.readlines()
self.info = {}
for i in range(len(LINE_lst)):
LINE_lst[i] = LINE_lst[i].strip(' \n\t')
LINE_lst[i] = remove_html_tags(LINE_lst[i])
lines = LINE_lst[:]
for line in lines:
if len(line) == 0:
LINE_lst.remove(line)
self.body = ' '.join(LINE_lst[1:])
self.title = LINE_lst[0]
f.close()
def __str__(self):
return self.title + '\n' + self.body
def __repr__(self):
return self.title
def __eq__(self,other):
n = lev(self.body,other.body)
k = len(self.body)
m = len(other.body)
return float(n)/max(k,m) <= 0.15
def __lt__(self,other):
return self.title < other.title
""" A Class that crawls the web """
class Crawler:
def __init__(self, directory):
self.folder = directory
def crawl(self):
pages = [f for f in os.listdir(self.folder) if f.endswith('.html')]
final_list = []
for i in range(len(pages)):
pages[i] = WebPage(self.folder + '\\' + pages[i])
pages[i].process()
for k in range(len(final_list)+1):
if k == len(final_list):
final_list.append(pages[i])
elif pages[i] == final_list[k]:
if pages[i] < final_list[k]:
final_list.append(pages[i])
final_list.remove(final_list[k])
break
print final_list
self.pages = final_list
everything works fine besides this freaking line final_list.remove(final_list[k]). help please? whats wrong here?

Answer 1

I'm not sure why your code doesn't work, it's difficult to test it because I don't know what kind of input should end up calling remove().
I suggest following these steps:
Make sure that remove() is called at some point.
remove() relies on your __eq__() method to find the item to remove, so make sure that __eq__() isn't the culprit.
As a side note, you will probably want to replace this:
self.folder + '\\' + pages[i]
with:
import os.path
# ...
os.path.join(self.folder, page[i])
This simple change should make your script work on all operating systems, rather than on Windows only. (GNU/Linux, Mac OS and other Unix-like OS use “/” as path separator.)
Please also consider replacing loops of this form:
for i in range(len(sequence)):
# Do something with sequence[i]
with:
for item in sequence:
# Do something with item
If you need the item index, use enumerate():
for i, item in enumerate(sequence):