""" Test cases for deltalakeopsserver/logstats/objstore.py - Cloud implementations (S3, GCS, AzureBlob) """ import os import sys import tempfile import shutil import unittest import gzip from unittest.mock import patch, MagicMock, Mock from io import BytesIO from botocore.exceptions import ClientError # Add project root to path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) from deltalakeopsserver.logstats.objstore import S3, GCS, AzureBlob class TestS3(unittest.TestCase): """Test S3 implementation with mocked boto3""" def setUp(self): """Set up test fixtures""" self.temp_dir = tempfile.mkdtemp() def tearDown(self): """Clean up test fixtures""" if os.path.exists(self.temp_dir): shutil.rmtree(self.temp_dir) # Clean up environment variables if 'AWS_ENDPOINT_S3' in os.environ: del os.environ['AWS_ENDPOINT_S3'] @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_initialization_default(self, mock_boto3): """Test S3 initialization with default config""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource s3 = S3() self.assertEqual(s3.s3, mock_s3_resource) mock_boto3.resource.assert_called_once_with('s3', endpoint_url=None, config=unittest.mock.ANY) @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_initialization_with_endpoint(self, mock_boto3): """Test S3 initialization with custom endpoint (AliCloud)""" os.environ['AWS_ENDPOINT_S3'] = 'https://oss-cn-hangzhou.aliyuncs.com' mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource s3 = S3() self.assertEqual(s3.s3, mock_s3_resource) # Should use custom endpoint and config call_args = mock_boto3.resource.call_args self.assertEqual(call_args[0][0], 's3') self.assertEqual(call_args[1]['endpoint_url'], 'https://oss-cn-hangzhou.aliyuncs.com') @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_list(self, mock_boto3): """Test S3 list method""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Mock bucket and objects mock_bucket = MagicMock() mock_obj1 = MagicMock() mock_obj1.key = 'prefix/file1.txt' mock_obj2 = MagicMock() mock_obj2.key = 'prefix/file2.txt' mock_bucket.objects.filter.return_value = [mock_obj1, mock_obj2] mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() result = s3.list('test-bucket', 'prefix/') self.assertEqual(len(result), 2) self.assertIn('prefix/file1.txt', result) self.assertIn('prefix/file2.txt', result) mock_bucket.objects.filter.assert_called_once_with(Prefix='prefix/') @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_object_exists(self, mock_boto3): """Test S3 object method when object exists""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Mock successful get mock_bucket = MagicMock() mock_object = MagicMock() mock_body = BytesIO(b'test content') mock_object.get.return_value = {'Body': mock_body} mock_bucket.Object.return_value = mock_object mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() result = s3.object('test-bucket', 'test-key') self.assertIsNotNone(result) self.assertEqual(result.read(), b'test content') @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_object_not_found(self, mock_boto3): """Test S3 object method when object does not exist""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Mock ClientError for 404 mock_bucket = MagicMock() mock_object = MagicMock() error_response = {'Error': {'Code': '404'}} mock_object.get.side_effect = ClientError(error_response, 'GetObject') mock_bucket.Object.return_value = mock_object mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() result = s3.object('test-bucket', 'test-key') self.assertIsNone(result) @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_object_gzip(self, mock_boto3): """Test S3 object method with gzip file""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Mock gzip content import gzip compressed_content = gzip.compress(b'test content') mock_body = BytesIO(compressed_content) mock_bucket = MagicMock() mock_object = MagicMock() mock_object.get.return_value = {'Body': mock_body} mock_bucket.Object.return_value = mock_object mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() result = s3.object('test-bucket', 'test-key.gz') self.assertIsNotNone(result) # Should return gzip file handle self.assertTrue(hasattr(result, 'read')) @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_upload(self, mock_boto3): """Test S3 upload method""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Create test file test_file = os.path.join(self.temp_dir, 'test.txt') with open(test_file, 'w') as f: f.write('test content') mock_bucket = MagicMock() mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() s3.upload('test-bucket', 'test-key', test_file) # Verify upload_fileobj was called mock_bucket.upload_fileobj.assert_called_once() call_args = mock_bucket.upload_fileobj.call_args # key is a positional argument, not keyword argument self.assertEqual(call_args[0][1], 'test-key') @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_upload_gzip(self, mock_boto3): """Test S3 upload method with gzip compression""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Create test file test_file = os.path.join(self.temp_dir, 'test.txt') with open(test_file, 'w') as f: f.write('test content') mock_bucket = MagicMock() mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() s3.upload('test-bucket', 'test-key.gz', test_file) # Verify upload_fileobj was called with gzip encoding mock_bucket.upload_fileobj.assert_called_once() call_args = mock_bucket.upload_fileobj.call_args self.assertEqual(call_args[1]['ExtraArgs']['ContentEncoding'], 'gzip') @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_download(self, mock_boto3): """Test S3 download method""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Mock download_file mock_bucket = MagicMock() def mock_download_file(key, filepath): with open(filepath, 'wb') as f: f.write(b'test content') mock_bucket.download_file.side_effect = mock_download_file mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() dest_file = os.path.join(self.temp_dir, 'downloaded.txt') result = s3.download('test-bucket', 'test-key', dest_file) self.assertTrue(result) self.assertTrue(os.path.exists(dest_file)) with open(dest_file, 'r') as f: self.assertEqual(f.read(), 'test content') @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_download_not_found(self, mock_boto3): """Test S3 download method when object does not exist""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Mock ClientError for 404 mock_bucket = MagicMock() error_response = {'Error': {'Code': '404'}} mock_bucket.download_file.side_effect = ClientError(error_response, 'GetObject') mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() dest_file = os.path.join(self.temp_dir, 'downloaded.txt') result = s3.download('test-bucket', 'test-key', dest_file) self.assertFalse(result) @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_existed(self, mock_boto3): """Test S3 existed method""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Mock successful load mock_bucket = MagicMock() mock_object = MagicMock() mock_object.load.return_value = None mock_bucket.Object.return_value = mock_object mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() result = s3.existed('test-bucket', 'test-key') self.assertTrue(result) mock_object.load.assert_called_once() @patch('deltalakeopsserver.logstats.objstore.boto3') def test_s3_existed_not_found(self, mock_boto3): """Test S3 existed method when object does not exist""" mock_s3_resource = MagicMock() mock_boto3.resource.return_value = mock_s3_resource # Mock ClientError for 404 mock_bucket = MagicMock() mock_object = MagicMock() error_response = {'Error': {'Code': '404'}} mock_object.load.side_effect = ClientError(error_response, 'HeadObject') mock_bucket.Object.return_value = mock_object mock_s3_resource.Bucket.return_value = mock_bucket s3 = S3() result = s3.existed('test-bucket', 'test-key') self.assertFalse(result) class TestGCS(unittest.TestCase): """Test GCS implementation with mocked google.cloud.storage""" def setUp(self): """Set up test fixtures""" self.temp_dir = tempfile.mkdtemp() def tearDown(self): """Clean up test fixtures""" if os.path.exists(self.temp_dir): shutil.rmtree(self.temp_dir) @patch('deltalakeopsserver.logstats.objstore.Client') def test_gcs_initialization(self, mock_client_class): """Test GCS initialization""" mock_client = MagicMock() mock_client_class.return_value = mock_client gcs = GCS() self.assertEqual(gcs.client, mock_client) mock_client_class.assert_called_once() @patch('deltalakeopsserver.logstats.objstore.Client') def test_gcs_list(self, mock_client_class): """Test GCS list method""" mock_client = MagicMock() mock_client_class.return_value = mock_client # Mock blobs mock_blob1 = MagicMock() mock_blob1.name = 'prefix/file1.txt' mock_blob2 = MagicMock() mock_blob2.name = 'prefix/file2.txt' mock_client.list_blobs.return_value = [mock_blob1, mock_blob2] gcs = GCS() result = gcs.list('test-bucket', 'prefix/') self.assertEqual(len(result), 2) self.assertIn('prefix/file1.txt', result) self.assertIn('prefix/file2.txt', result) mock_client.list_blobs.assert_called_once_with('test-bucket', prefix='prefix/') @patch('deltalakeopsserver.logstats.objstore.Client') def test_gcs_object_exists(self, mock_client_class): """Test GCS object method when object exists""" mock_client = MagicMock() mock_client_class.return_value = mock_client # Mock bucket and blob mock_bucket = MagicMock() mock_blob = MagicMock() mock_blob.download_as_bytes.return_value = b'test content' mock_client.get_bucket.return_value = mock_bucket with patch('deltalakeopsserver.logstats.objstore.Blob', return_value=mock_blob): gcs = GCS() result = gcs.object('test-bucket', 'test-key') self.assertIsNotNone(result) self.assertIsInstance(result, BytesIO) result.seek(0) self.assertEqual(result.read(), b'test content') @patch('deltalakeopsserver.logstats.objstore.Client') def test_gcs_object_not_found(self, mock_client_class): """Test GCS object method when object does not exist""" from google.cloud.exceptions import NotFound mock_client = MagicMock() mock_client_class.return_value = mock_client # Mock NotFound exception mock_bucket = MagicMock() mock_blob = MagicMock() mock_blob.download_as_bytes.side_effect = NotFound('Object not found') mock_client.get_bucket.return_value = mock_bucket with patch('deltalakeopsserver.logstats.objstore.Blob', return_value=mock_blob): gcs = GCS() result = gcs.object('test-bucket', 'test-key') self.assertIsNone(result) @patch('deltalakeopsserver.logstats.objstore.Client') def test_gcs_upload(self, mock_client_class): """Test GCS upload method""" mock_client = MagicMock() mock_client_class.return_value = mock_client # Create test file test_file = os.path.join(self.temp_dir, 'test.txt') with open(test_file, 'w') as f: f.write('test content') mock_bucket = MagicMock() mock_blob = MagicMock() mock_client.get_bucket.return_value = mock_bucket with patch('deltalakeopsserver.logstats.objstore.Blob', return_value=mock_blob): gcs = GCS() gcs.upload('test-bucket', 'test-key', test_file) mock_blob.upload_from_filename.assert_called_once_with(test_file) @patch('deltalakeopsserver.logstats.objstore.Client') def test_gcs_download(self, mock_client_class): """Test GCS download method""" mock_client = MagicMock() mock_client_class.return_value = mock_client # Mock download mock_bucket = MagicMock() mock_blob = MagicMock() def mock_download_to_filename(filepath): with open(filepath, 'wb') as f: f.write(b'test content') mock_blob.download_to_filename.side_effect = mock_download_to_filename mock_client.get_bucket.return_value = mock_bucket with patch('deltalakeopsserver.logstats.objstore.Blob', return_value=mock_blob): gcs = GCS() dest_file = os.path.join(self.temp_dir, 'downloaded.txt') result = gcs.download('test-bucket', 'test-key', dest_file) self.assertTrue(result) self.assertTrue(os.path.exists(dest_file)) @patch('deltalakeopsserver.logstats.objstore.Client') def test_gcs_download_not_found(self, mock_client_class): """Test GCS download method when object does not exist""" from google.cloud.exceptions import NotFound mock_client = MagicMock() mock_client_class.return_value = mock_client # Mock NotFound exception mock_bucket = MagicMock() mock_blob = MagicMock() mock_blob.download_to_filename.side_effect = NotFound('Object not found') mock_client.get_bucket.return_value = mock_bucket with patch('deltalakeopsserver.logstats.objstore.Blob', return_value=mock_blob): gcs = GCS() dest_file = os.path.join(self.temp_dir, 'downloaded.txt') result = gcs.download('test-bucket', 'test-key', dest_file) self.assertFalse(result) @patch('deltalakeopsserver.logstats.objstore.Client') def test_gcs_existed(self, mock_client_class): """Test GCS existed method""" mock_client = MagicMock() mock_client_class.return_value = mock_client # Mock successful reload mock_bucket = MagicMock() mock_blob = MagicMock() mock_blob.reload.return_value = None mock_client.get_bucket.return_value = mock_bucket with patch('deltalakeopsserver.logstats.objstore.Blob', return_value=mock_blob): gcs = GCS() result = gcs.existed('test-bucket', 'test-key') self.assertTrue(result) mock_blob.reload.assert_called_once() @patch('deltalakeopsserver.logstats.objstore.Client') def test_gcs_existed_not_found(self, mock_client_class): """Test GCS existed method when object does not exist""" from google.cloud.exceptions import NotFound mock_client = MagicMock() mock_client_class.return_value = mock_client # Mock NotFound exception mock_bucket = MagicMock() mock_blob = MagicMock() mock_blob.reload.side_effect = NotFound('Object not found') mock_client.get_bucket.return_value = mock_bucket with patch('deltalakeopsserver.logstats.objstore.Blob', return_value=mock_blob): gcs = GCS() result = gcs.existed('test-bucket', 'test-key') self.assertFalse(result) class TestAzureBlob(unittest.TestCase): """Test AzureBlob implementation with mocked azure.storage.blob""" def setUp(self): """Set up test fixtures""" self.temp_dir = tempfile.mkdtemp() os.environ['STORAGE_ACCOUNT_NAME'] = 'test-account' def tearDown(self): """Clean up test fixtures""" if os.path.exists(self.temp_dir): shutil.rmtree(self.temp_dir) if 'STORAGE_ACCOUNT_NAME' in os.environ: del os.environ['STORAGE_ACCOUNT_NAME'] @patch('deltalakeopsserver.logstats.objstore.DefaultAzureCredential') @patch('deltalakeopsserver.logstats.objstore.BlobServiceClient') def test_azure_blob_initialization(self, mock_blob_service_client_class, mock_credential_class): """Test AzureBlob initialization""" mock_credential = MagicMock() mock_credential_class.return_value = mock_credential mock_blob_service_client = MagicMock() mock_blob_service_client_class.return_value = mock_blob_service_client azure_blob = AzureBlob() self.assertEqual(azure_blob.blob_service_client, mock_blob_service_client) mock_blob_service_client_class.assert_called_once() @patch('deltalakeopsserver.logstats.objstore.DefaultAzureCredential') @patch('deltalakeopsserver.logstats.objstore.BlobServiceClient') def test_azure_blob_list(self, mock_blob_service_client_class, mock_credential_class): """Test AzureBlob list method""" mock_credential = MagicMock() mock_credential_class.return_value = mock_credential mock_blob_service_client = MagicMock() mock_blob_service_client_class.return_value = mock_blob_service_client # Mock container client and blobs mock_container_client = MagicMock() mock_blob1 = MagicMock() mock_blob1.name = 'prefix/file1.txt' mock_blob2 = MagicMock() mock_blob2.name = 'prefix/file2.txt' mock_container_client.list_blobs.return_value = [mock_blob1, mock_blob2] mock_blob_service_client.get_container_client.return_value = mock_container_client azure_blob = AzureBlob() result = azure_blob.list('test-container', 'prefix/') self.assertEqual(len(result), 2) self.assertIn('prefix/file1.txt', result) self.assertIn('prefix/file2.txt', result) mock_container_client.list_blobs.assert_called_once_with(name_starts_with='prefix/') @patch('deltalakeopsserver.logstats.objstore.DefaultAzureCredential') @patch('deltalakeopsserver.logstats.objstore.BlobServiceClient') def test_azure_blob_object_exists(self, mock_blob_service_client_class, mock_credential_class): """Test AzureBlob object method when object exists""" mock_credential = MagicMock() mock_credential_class.return_value = mock_credential mock_blob_service_client = MagicMock() mock_blob_service_client_class.return_value = mock_blob_service_client # Mock blob client and download mock_blob_client = MagicMock() mock_download_blob = MagicMock() stream = BytesIO() stream.write(b'test content') stream.seek(0) mock_download_blob.readinto.return_value = None mock_blob_client.download_blob.return_value = mock_download_blob mock_blob_service_client.get_blob_client.return_value = mock_blob_client azure_blob = AzureBlob() result = azure_blob.object('test-container', 'test-key') self.assertIsNotNone(result) self.assertIsInstance(result, BytesIO) @patch('deltalakeopsserver.logstats.objstore.DefaultAzureCredential') @patch('deltalakeopsserver.logstats.objstore.BlobServiceClient') def test_azure_blob_object_not_found(self, mock_blob_service_client_class, mock_credential_class): """Test AzureBlob object method when object does not exist""" from azure.core.exceptions import ResourceNotFoundError mock_credential = MagicMock() mock_credential_class.return_value = mock_credential mock_blob_service_client = MagicMock() mock_blob_service_client_class.return_value = mock_blob_service_client # Mock ResourceNotFoundError mock_blob_client = MagicMock() mock_blob_client.download_blob.side_effect = ResourceNotFoundError('Blob not found') mock_blob_service_client.get_blob_client.return_value = mock_blob_client azure_blob = AzureBlob() result = azure_blob.object('test-container', 'test-key') self.assertIsNone(result) @patch('deltalakeopsserver.logstats.objstore.DefaultAzureCredential') @patch('deltalakeopsserver.logstats.objstore.BlobServiceClient') def test_azure_blob_upload(self, mock_blob_service_client_class, mock_credential_class): """Test AzureBlob upload method""" mock_credential = MagicMock() mock_credential_class.return_value = mock_credential mock_blob_service_client = MagicMock() mock_blob_service_client_class.return_value = mock_blob_service_client # Create test file test_file = os.path.join(self.temp_dir, 'test.txt') with open(test_file, 'w') as f: f.write('test content') mock_blob_client = MagicMock() mock_blob_service_client.get_blob_client.return_value = mock_blob_client azure_blob = AzureBlob() azure_blob.upload('test-container', 'test-key', test_file) mock_blob_client.upload_blob.assert_called_once() call_args = mock_blob_client.upload_blob.call_args self.assertEqual(call_args[1]['overwrite'], True) @patch('deltalakeopsserver.logstats.objstore.DefaultAzureCredential') @patch('deltalakeopsserver.logstats.objstore.BlobServiceClient') def test_azure_blob_download(self, mock_blob_service_client_class, mock_credential_class): """Test AzureBlob download method""" mock_credential = MagicMock() mock_credential_class.return_value = mock_credential mock_blob_service_client = MagicMock() mock_blob_service_client_class.return_value = mock_blob_service_client # Mock download mock_blob_client = MagicMock() mock_download_blob = MagicMock() def mock_readinto(file): file.write(b'test content') mock_download_blob.readinto.side_effect = mock_readinto mock_blob_client.download_blob.return_value = mock_download_blob mock_blob_service_client.get_blob_client.return_value = mock_blob_client azure_blob = AzureBlob() dest_file = os.path.join(self.temp_dir, 'downloaded.txt') result = azure_blob.download('test-container', 'test-key', dest_file) self.assertTrue(result) self.assertTrue(os.path.exists(dest_file)) with open(dest_file, 'r') as f: self.assertEqual(f.read(), 'test content') @patch('deltalakeopsserver.logstats.objstore.DefaultAzureCredential') @patch('deltalakeopsserver.logstats.objstore.BlobServiceClient') def test_azure_blob_download_not_found(self, mock_blob_service_client_class, mock_credential_class): """Test AzureBlob download method when object does not exist""" from azure.core.exceptions import ResourceNotFoundError mock_credential = MagicMock() mock_credential_class.return_value = mock_credential mock_blob_service_client = MagicMock() mock_blob_service_client_class.return_value = mock_blob_service_client # Mock ResourceNotFoundError mock_blob_client = MagicMock() mock_blob_client.download_blob.side_effect = ResourceNotFoundError('Blob not found') mock_blob_service_client.get_blob_client.return_value = mock_blob_client azure_blob = AzureBlob() dest_file = os.path.join(self.temp_dir, 'downloaded.txt') result = azure_blob.download('test-container', 'test-key', dest_file) self.assertFalse(result) @patch('deltalakeopsserver.logstats.objstore.DefaultAzureCredential') @patch('deltalakeopsserver.logstats.objstore.BlobServiceClient') def test_azure_blob_existed(self, mock_blob_service_client_class, mock_credential_class): """Test AzureBlob existed method""" mock_credential = MagicMock() mock_credential_class.return_value = mock_credential mock_blob_service_client = MagicMock() mock_blob_service_client_class.return_value = mock_blob_service_client # Mock successful get_blob_properties mock_blob_client = MagicMock() mock_blob_client.get_blob_properties.return_value = MagicMock() mock_blob_service_client.get_blob_client.return_value = mock_blob_client azure_blob = AzureBlob() result = azure_blob.existed('test-container', 'test-key') self.assertTrue(result) mock_blob_client.get_blob_properties.assert_called_once() @patch('deltalakeopsserver.logstats.objstore.DefaultAzureCredential') @patch('deltalakeopsserver.logstats.objstore.BlobServiceClient') def test_azure_blob_existed_not_found(self, mock_blob_service_client_class, mock_credential_class): """Test AzureBlob existed method when object does not exist""" from azure.core.exceptions import ResourceNotFoundError mock_credential = MagicMock() mock_credential_class.return_value = mock_credential mock_blob_service_client = MagicMock() mock_blob_service_client_class.return_value = mock_blob_service_client # Mock ResourceNotFoundError mock_blob_client = MagicMock() mock_blob_client.get_blob_properties.side_effect = ResourceNotFoundError('Blob not found') mock_blob_service_client.get_blob_client.return_value = mock_blob_client azure_blob = AzureBlob() result = azure_blob.existed('test-container', 'test-key') self.assertFalse(result) if __name__ == '__main__': unittest.main()