Skip to content

Commit

Permalink
update log2csv
Browse files Browse the repository at this point in the history
  • Loading branch information
Suzhou65 committed Aug 11, 2023
1 parent a5c2b3e commit 357ae36
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 11 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
__pycache__/
.pyc
.DS_Store
34 changes: 23 additions & 11 deletions lib_gist/python_pixmicat_log2csv.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
## Processing Pixmicat Logfile
Convert Pixmicat! .logf to .csv format.
Convert Pixmicat! log file into CSV format.
### Import Module
```python
# -*- coding: utf-8 -*-
Expand All @@ -10,8 +10,8 @@ import pandas as pd
```python
def filename_check(list_result):
# String "pixmicat_log" setting as you logfile configuration.
if list_result.find("pixmicat_log") != -1:
return True
if list_result.find("audit") != -1:
return True
else:
return False
```
Expand All @@ -21,12 +21,12 @@ def read_logfile(input_filename):
try:
# Trans to pandas dataframe
# Header 'E-Series' is contain extra
header = ["IP","TIME","ACTION","CONTENT","E1","E2","E3","E4","E5","E6","E7","E8","E9","E10"]
log_file = pd.read_csv(input_filename, encoding="utf-8", header=None, sep="[[\]]", names=header, engine="python")
header = ["IP","TIME","ACTION","CONTENT","E1","E2","E3","E4","E5","E6","E7","E8","E9","E10","E11","EX12"]
log_file = pd.read_csv(input_filename, encoding="utf-8", header=None, sep="[\[\]]", names=header, engine="python")
# Merge
log_file["CONTENTS"] = log_file[log_file.columns[3:]].apply(lambda x: " ".join(x.dropna().astype(str)), axis=1)
# Drop
log_file= log_file.drop(columns=["CONTENT","E1","E2","E3","E4","E5","E6","E7","E8","E9","E10"])
log_file= log_file.drop(columns=["CONTENT","E1","E2","E3","E4","E5","E6","E7","E8","E9","E10","E11","EX12"])
# Filename
if input_filename.find("txt") == -1:
output_filename = input_filename.replace("ht","").replace("_audit","") + (".csv")
Expand All @@ -37,18 +37,15 @@ def read_logfile(input_filename):
log_file.to_csv(output_filename, encoding="utf_8_sig", index=False)
# Print Info when translation complete
print([input_filename, "Translation Complete"])
return True
# Error handling
# Error handling
except FileNotFoundError:
print([input_filename, "SKip"])
return False
except Exception as error:
print([input_filename, error])
return False
```
### Using
```python
# Scanning local folder, result as list
# Ask list
list_asking = subprocess.Popen(["ls", "-ls"], stdout=subprocess.PIPE)
# AWK filter
filter_awk = ["awk", "{print $10}"]
Expand All @@ -71,4 +68,19 @@ for input_filename in input_filenames:
```python
input_filename = input()
read_logfile(input_filename)
```

### FutureWarning issue
You may get this warning
```
/opt/homebrew/lib/python3.9/site-packages/pandas/io/parsers/python_parser.py:222: FutureWarning: Possible nested set at position 1
pat = re.compile(sep)
```
Fix this
```python
log_file = pd.read_csv(input_filename, encoding="utf-8", header=None, sep="[[\]]", names=header, engine="python")
```
To this
```python
log_file = pd.read_csv(input_filename, encoding="utf-8", header=None, sep="[\[\]]", names=header, engine="python")
```

0 comments on commit 357ae36

Please sign in to comment.