Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Displaying only my custom bucket #107

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Displaying only my custom bucket #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
66ef7e4
Displaying only my custom bucket
adhithya-nair Oct 19, 2023
edf3d18
Changes in package.json
adhithya-nair Oct 19, 2023
d9a5cbf
Package Number
adhithya-nair Oct 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 68 additions & 23 deletions jupyterlab_s3_browser/handlers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
"""
import base64
import json
import os
import logging
from .logger import log
from pathlib import Path

import boto3
Expand All @@ -13,6 +15,11 @@
from jupyter_server.base.handlers import APIHandler
from jupyter_server.utils import url_path_join

for key, value in os.environ.items():
print(f'{key}={value}')
myCustomBucket = os.getenv('S3_BROWSER_BUCKET')
print(f"My Custom Bucket -> {myCustomBucket}")


class DirectoryNotEmptyException(Exception):
"""Raise for attempted deletions of non-empty directories"""
Expand All @@ -21,9 +28,9 @@ class DirectoryNotEmptyException(Exception):


def create_s3fs(config):

print("inside create_s3fs")
if config.endpoint_url and config.client_id and config.client_secret:

print(f"key -> {config.client_id} , secret -> {config.client_secret} , token -> {config.session_token} , client_kwargs-> {config.endpoint_url}")
return s3fs.S3FileSystem(
key=config.client_id,
secret=config.client_secret,
Expand All @@ -36,7 +43,7 @@ def create_s3fs(config):


def create_s3_resource(config):

print("inside create_s3_resource")
if config.endpoint_url and config.client_id and config.client_secret:

return boto3.resource(
Expand All @@ -55,6 +62,7 @@ def _test_aws_s3_role_access():
"""
Checks if we have access to AWS S3 through role-based access
"""
print("inside _test_aws_s3_role_access")
test = boto3.resource("s3")
all_buckets = test.buckets.all()
result = [
Expand All @@ -68,7 +76,7 @@ def has_aws_s3_role_access():
"""
Returns true if the user has access to an aws S3 bucket
"""

print("inside has_aws_s3_role_access")
# avoid making requests to AWS if the user's ~/.aws/credentials file has credentials for a different provider,
# e.g. https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-aws-cli#aws-cli-config
aws_credentials_file = Path("{}/.aws/credentials".format(Path.home()))
Expand All @@ -82,7 +90,7 @@ def has_aws_s3_role_access():
"AKIA"
) and not access_key_id.startswith("ASIA"):
# if any keys are not valid AWS keys, don't try to authenticate
logging.info(
log.info(
"Found invalid AWS aws_access_key_id in ~/.aws/credentials file, "
"will not attempt to authenticate through ~/.aws/credentials."
)
Expand All @@ -94,7 +102,7 @@ def has_aws_s3_role_access():
except NoCredentialsError:
return False
except Exception as e:
logging.error(e)
log.error(e)
return False


Expand All @@ -103,6 +111,7 @@ def test_s3_credentials(endpoint_url, client_id, client_secret, session_token):
Checks if we're able to list buckets with these credentials.
If not, it throws an exception.
"""
print("inside test_s3_credentials")
test = boto3.resource(
"s3",
aws_access_key_id=client_id,
Expand All @@ -111,12 +120,21 @@ def test_s3_credentials(endpoint_url, client_id, client_secret, session_token):
aws_session_token=session_token,
)
all_buckets = test.buckets.all()
logging.debug(
[
{"name": bucket.name + "/", "path": bucket.name + "/", "type": "directory"}
for bucket in all_buckets
]
)
# logging.debug(
# [
# {"name": bucket.name + "/", "path": bucket.name + "/", "type": "directory"}
# for bucket in all_buckets
# ]
# )

# data = [
# {"name": bucket.name + "/", "path": bucket.name + "/", "type": "directory"}
# for bucket in all_buckets
# ]

print("Creating connection to my bucket")
print(f"List of buckets -> {all_buckets}")
# print(json.dumps(data, indent=4))


class AuthHandler(APIHandler): # pylint: disable=abstract-method
Expand All @@ -126,6 +144,7 @@ class AuthHandler(APIHandler): # pylint: disable=abstract-method

@property
def config(self):
print("inside AuthHandler -> config")
return self.settings["s3_config"]

@tornado.web.authenticated
Expand All @@ -134,9 +153,13 @@ def get(self, path=""):
Checks if the user is already authenticated
against an s3 instance.
"""
print("inside AuthHandler -> get")
print(self.settings["s3_config"])
print(dir(self.settings["s3_config"]))
authenticated = False
if has_aws_s3_role_access():
authenticated = True
print(authenticated)

if not authenticated:

Expand All @@ -149,7 +172,8 @@ def get(self, path=""):
config.client_secret,
config.session_token,
)
logging.debug("...successfully authenticated")
print("")
log.info("...successfully authenticated")

# If no exceptions were encountered during testS3Credentials,
# then assume we're authenticated
Expand All @@ -159,8 +183,8 @@ def get(self, path=""):
# If an exception was encountered,
# assume that we're not yet authenticated
# or invalid credentials were provided
logging.debug("...failed to authenticate")
logging.debug(err)
log.info("...failed to authenticate")
log.error(err)

self.finish(json.dumps({"authenticated": authenticated}))

Expand All @@ -169,7 +193,7 @@ def post(self, path=""):
"""
Sets s3 credentials.
"""

print("inside AuthHandler -> post")
try:
req = json.loads(self.request.body)
endpoint_url = req["endpoint_url"]
Expand All @@ -186,7 +210,7 @@ def post(self, path=""):

self.finish(json.dumps({"success": True}))
except Exception as err:
logging.info("unable to authenticate using credentials")
log.info("unable to authenticate using credentials")
self.finish(json.dumps({"success": False, "message": str(err)}))


Expand Down Expand Up @@ -220,6 +244,7 @@ def get(self, path=""):
Takes a path and returns lists of files/objects
and directories/prefixes based on the path.
"""
print("inside S3Handler -> get")
path = path[1:]
# logging.info("GET {}".format(path))

Expand All @@ -238,19 +263,37 @@ def get(self, path=""):
"type": "file",
"content": base64.encodebytes(f.read()).decode("ascii"),
}
print("inside if")
print(result)
else:
raw_result = list(
map(convertS3FStoJupyterFormat, self.s3fs.listdir(path))
)
result = list(filter(lambda x: x["name"] != "", raw_result))
print(f"Raw result -> {raw_result}")
print(f"Length of raw_result -> {len(raw_result)}")

checkBucket = raw_result[0]['path'].split('/')[0]
print(f"Check bucket name for filtering -> {checkBucket}")

if checkBucket != myCustomBucket:
filtered_result = [item for item in raw_result if item['name'] == myCustomBucket]
result = list(filter(lambda x: x["name"] != "", filtered_result))

else:
result = list(filter(lambda x: x["name"] != "", raw_result))

# print(f"Filtered List -> {filtered_result}")
# result = list(filter(lambda x: x["name"] != "", filtered_result))
print("inside else")
print(result)

except S3ResourceNotFoundException as e:
result = {
"error": 404,
"message": "The requested resource could not be found.",
}
except Exception as e:
logging.error("Exception encountered during GET {}: {}".format(path, e))
log.error("Exception encountered during GET {}: {}".format(path, e))
result = {"error": 500, "message": str(e)}

self.finish(json.dumps(result))
Expand All @@ -261,6 +304,7 @@ def put(self, path=""):
Takes a path and returns lists of files/objects
and directories/prefixes based on the path.
"""
print("inside S3Handler -> put")
path = path[1:]

result = {}
Expand Down Expand Up @@ -323,7 +367,7 @@ def put(self, path=""):
"message": "The requested resource could not be found.",
}
except Exception as e:
logging.error(e)
log.error(e)
result = {"error": 500, "message": str(e)}

self.finish(json.dumps(result))
Expand All @@ -334,6 +378,7 @@ def delete(self, path=""):
Takes a path and returns lists of files/objects
and directories/prefixes based on the path.
"""
print("inside S3Handler -> delete")
path = path[1:]
# logging.info("DELETE: {}".format(path))

Expand Down Expand Up @@ -372,7 +417,7 @@ def delete(self, path=""):
self.s3fs.rm(path)

except S3ResourceNotFoundException as e:
logging.error(e)
log.error(e)
result = {
"error": 404,
"message": "The requested resource could not be found.",
Expand All @@ -381,8 +426,8 @@ def delete(self, path=""):
# logging.info("Attempted to delete non-empty directory")
result = {"error": 400, "error": "DIR_NOT_EMPTY"}
except Exception as e:
logging.error("error while deleting")
logging.error(e)
log.error("error while deleting")
log.error(e)
result = {"error": 500, "message": str(e)}

self.finish(json.dumps(result))
Expand Down
54 changes: 54 additions & 0 deletions jupyterlab_s3_browser/logger.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import datetime
import json
import logging
import os
import sys

class CustomFormatter(logging.Formatter):

grey = "\x1b[38;20m"
darkGrey = '\033[1;30m'
yellow = "\x1b[33;20m"
red = "\x1b[31;20m"
boldRed = "\x1b[31;1m"
green = '\033[0;32m'
blue = '\033[0;34m'
purple = '\033[0;35m'
reset = "\x1b[0m"
if os.getenv("NOTEBOOK", None) or os.getenv("API_ENV", None):
format = '[%(asctime)s] [%(levelname)s] %(message)s'
else:
format = '[%(asctime)s] [%(levelname)s] [%(funcName)s] [%(lineno)d] %(message)s'

FORMATS = {
logging.DEBUG: green + format + reset,
logging.INFO: blue + format + reset,
logging.WARNING: yellow + format + reset,
logging.ERROR: boldRed + format + reset,
logging.CRITICAL: boldRed + format + reset
}

def format(self, record):

log_fmt = self.FORMATS.get(record.levelno)
formatter = logging.Formatter(log_fmt)
return formatter.format(record)

class Logger:

def __init__(self):
logs = logging.getLogger(__name__)
logs.propagate = False
logs.setLevel(10 if os.getenv("VERBOSE") == "True" else 20)
handler_stream = logging.StreamHandler(sys.stdout)
handler_stream.setFormatter(CustomFormatter())
logs.addHandler(handler_stream)
self.DataLog = logs

def getLogger(self):
return self.DataLog




log = Logger().DataLog
8 changes: 4 additions & 4 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "jupyterlab-s3-browser",
"version": "0.12.0",
"version": "1.0",
"description": "JupyterLab extension for browsing S3-compatible object storage",
"keywords": [
"s3",
Expand All @@ -11,9 +11,9 @@
"jupyterlab",
"jupyterlab-extension"
],
"homepage": "https://github.com/IBM/jupyterlab_s3_browser",
"homepage": "https://github.com/adhithya10/jupyterlab-s3-browser",
"bugs": {
"url": "https://github.com/IBM/jupyterlab_s3_browser/issues"
"url": "https://github.com/adhithya10/jupyterlab-s3-browser"
},
"license": "Apache-2.0",
"author": "James Reeve",
Expand All @@ -31,7 +31,7 @@
},
"repository": {
"type": "git",
"url": "https://github.com/IBM/jupyterlab-s3-browser.git"
"url": "https://github.com/adhithya10/jupyterlab-s3-browser.git"
},
"scripts": {
"build": "jlpm run build:lib && jlpm run build:labextension:dev",
Expand Down